Merge pull request #112 from JakduK/batch

엘라스틱서치 업그레이드 및 초기화 배치 추가

build.gradle

@@ -52,8 +52,8 @@ subprojects {
         compile group: 'commons-fileupload', name: 'commons-fileupload', version:'1.3.2'
         compile group: 'commons-io', name: 'commons-io', version:'2.4'
         compile group: 'rome', name: 'rome', version:'1.0'
-        compile group: 'org.elasticsearch', name: 'elasticsearch', version:'1.7.5'
-        compile group: 'io.searchbox', name: 'jest', version:'2.0.2'
+        compile group: 'org.elasticsearch', name: 'elasticsearch', version:'2.4.0'
+        compile group: 'io.searchbox', name: 'jest', version:'2.0.3'
         compile group: 'net.gpedro.integrations.slack', name: 'slack-webhook', version:'1.1.1'
         compile group: 'javax.mail', name: 'mail', version:'1.4.7'
         compile 'commons-beanutils:commons-beanutils:1.9.2'

jakduk-api/src/main/java/com/jakduk/api/restcontroller/admin/AdminRestController.java

@@ -10,6 +10,8 @@
 import java.util.Objects;
 import javax.annotation.Resource;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.jakduk.core.service.*;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
 import lombok.extern.slf4j.Slf4j;
@@ -45,10 +47,6 @@
 import com.jakduk.core.model.web.ThumbnailSizeWrite;
 import com.jakduk.core.model.web.jakdu.JakduScheduleGroupWrite;
 import com.jakduk.core.model.web.jakdu.JakduScheduleWrite;
-import com.jakduk.core.service.AdminService;
-import com.jakduk.core.service.CommonService;
-import com.jakduk.core.service.CompetitionService;
-import com.jakduk.core.service.StatsService;
 
 /**
  * @author pyohwan
@@ -76,6 +74,9 @@ public class AdminRestController {
 	@Autowired
 	private CompetitionService competitionService;
 
+	@Autowired
+	private SearchService searchService;
+
 	@ApiOperation(value = "알림판 목록")
 	@RequestMapping(value = "/home/descriptions", method = RequestMethod.GET)
 	public Map<String, Object> getHomeDescriptions() {
@@ -632,19 +633,19 @@ public Map<String, Object> initBoardCategory() {
 	@ApiOperation(value = "검색 인덱스 초기화")
 	@RequestMapping(value = "/search/index/init", method = RequestMethod.POST)
 	public Map<String, Object> initSearchIndex() {
-		return adminService.initSearchIndex();
+		return searchService.initSearchIndex();
 	}
 
 	@ApiOperation(value = "검색 타입 초기화")
 	@RequestMapping(value = "/search/type/init", method = RequestMethod.POST)
-	public Map<String, Object> initSearchType() {
-		return adminService.initSearchType();
+	public Map<String, Object> initSearchType() throws JsonProcessingException {
+		return searchService.initSearchType();
 	}
 
 	@ApiOperation(value = "검색 데이터 초기화")
 	@RequestMapping(value = "/search/data/init", method = RequestMethod.POST)
 	public Map<String, Object> initSearchData() {
-		return adminService.initSearchData();
+		return searchService.initSearchDocuments();
 	}
 
 	@RequestMapping(value = "/thumbnail/size", method = RequestMethod.GET)

jakduk-batch/build.gradle

@@ -18,7 +18,7 @@ configurations {
 	all*.exclude group:'org.springframework', module:'spring-web'
 	all*.exclude group:'commons-fileupload', module:'commons-fileupload'
 	all*.exclude group:'commons-beanutils', module:'commons-beanutils'
-	all*.exclude group:'org.elasticsearch', module:'elasticsearch'
+	//all*.exclude group:'org.elasticsearch', module:'elasticsearch'
 	all*.exclude group:'org.hibernate.javax.persistence'
 }
 

jakduk-batch/src/main/java/com/jakduk/batch/ChangeBoardImageUrlBatchConfig.java → jakduk-batch/src/main/java/com/jakduk/batch/ChangeBoardImageUrlBatch.java

@@ -27,7 +27,7 @@
  */
 
 @Configuration
-public class ChangeBoardImageUrlBatchConfig {
+public class ChangeBoardImageUrlBatch {
 
 	@Autowired
 	private JobBuilderFactory jobBuilderFactory;

jakduk-batch/src/main/java/com/jakduk/batch/InitElasticsearchIndexBatch.java

@@ -0,0 +1,57 @@
+package com.jakduk.batch;
+
+import com.jakduk.core.service.SearchService;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.springframework.batch.core.Job;
+import org.springframework.batch.core.Step;
+import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
+import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
+import org.springframework.batch.core.launch.support.RunIdIncrementer;
+import org.springframework.batch.repeat.RepeatStatus;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * Created by pyohwan on 16. 9. 18.
+ */
+
+@Configuration
+public class InitElasticsearchIndexBatch {
+
+    @Autowired
+    private JobBuilderFactory jobBuilderFactory;
+
+    @Autowired
+    private StepBuilderFactory stepBuilderFactory;
+
+    @Autowired
+    private SearchService searchService;
+
+    @Bean
+    public Step step1() {
+        return stepBuilderFactory.get("initElasticsearchIndexStep")
+                .tasklet((contribution, chunkContext) -> {
+
+                    try {
+                        searchService.deleteIndex();
+                    } catch (IndexNotFoundException e) {
+                    }
+
+                    searchService.initSearchIndex();
+                    searchService.initSearchType();
+                    searchService.initSearchDocuments();
+
+                    return RepeatStatus.FINISHED;
+                })
+                .build();
+    }
+
+    @Bean
+    public Job job(Step step1) throws Exception {
+        return jobBuilderFactory.get("initElasticsearchIndexJob")
+                .incrementer(new RunIdIncrementer())
+                .start(step1)
+                .build();
+    }
+}

jakduk-batch/src/main/java/com/jakduk/batch/processor/ChangeBoardImageUrlProcessor.java

@@ -1,12 +1,17 @@
 package com.jakduk.batch.processor;
 
+import com.jakduk.core.common.CommonConst;
 import com.jakduk.core.model.db.BoardFree;
+import com.jakduk.core.model.etc.CommonCount;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.batch.item.ItemProcessor;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.env.Environment;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
 import java.util.stream.Stream;
 
 /**
@@ -50,9 +55,24 @@ public class ChangeBoardImageUrlProcessor implements ItemProcessor<BoardFree, Bo
 		if (StringUtils.contains(item.getContent(), beforeImageUrl01)) {
 			String newContent = StringUtils.replace(item.getContent(), beforeImageUrl01, afterImageUrl);
 			item.setContent(newContent);
+
+			List<CommonConst.BATCH_TYPE> batchList = Optional.ofNullable(item.getBatch()).orElseGet(ArrayList::new);
+
+			if (batchList.stream().noneMatch(batch -> batch.equals(CommonConst.BATCH_TYPE.CHANGE_BOARD_CONTENT_IMAGE_URL_01))) {
+				batchList.add(CommonConst.BATCH_TYPE.CHANGE_BOARD_CONTENT_IMAGE_URL_01);
+				item.setBatch(batchList);
+			}
+
 		} else if (StringUtils.contains(item.getContent(), beforeImageUrl02)) {
 			String newContent = StringUtils.replace(item.getContent(), beforeImageUrl02, afterImageUrl);
 			item.setContent(newContent);
+
+			List<CommonConst.BATCH_TYPE> batchList = Optional.ofNullable(item.getBatch()).orElseGet(ArrayList::new);
+
+			if (batchList.stream().noneMatch(batch -> batch.equals(CommonConst.BATCH_TYPE.CHANGE_BOARD_CONTENT_IMAGE_URL_01))) {
+				batchList.add(CommonConst.BATCH_TYPE.CHANGE_BOARD_CONTENT_IMAGE_URL_01);
+				item.setBatch(batchList);
+			}
 		}
 
 		log.debug("resultItemId=" + item.getId());

jakduk-core/src/main/java/com/jakduk/core/common/CommonConst.java

@@ -74,6 +74,9 @@ public enum BOARD_CATEGORY_TYPE {
 	public final static String RESPONSE_VOID_OBJECT = "{}";
 	public final static String RESPONSE_ERROR_DEFAULT_CODE = "-";
 
+	public final static String REGEX_FIND_HTML_TAG = "<(/)?([a-zA-Z0-9]*)(\\s[a-zA-Z0-9]*=[^>]*)?(\\s)*(/)?>";
+	public final static String REGEX_FIND_HTML_WHITESPACE = "\r|\n|&nbsp;";
+
 	/**
 	 * Social 프로바이더 종류.
 	 */

jakduk-core/src/main/java/com/jakduk/core/configuration/JestConfig.java

@@ -1,14 +1,25 @@
 package com.jakduk.core.configuration;
 
-import java.util.Arrays;
-
+import com.jakduk.core.exception.ServiceError;
+import com.jakduk.core.exception.ServiceException;
+import io.searchbox.client.JestClient;
+import io.searchbox.client.JestClientFactory;
+import io.searchbox.client.config.HttpClientConfig;
+import org.apache.commons.lang3.StringUtils;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
-import io.searchbox.client.JestClient;
-import io.searchbox.client.JestClientFactory;
-import io.searchbox.client.config.HttpClientConfig;
+import java.net.InetAddress;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.UnknownHostException;
+import java.util.Arrays;
+import java.util.stream.Stream;
 
 /**
 * @author <a href="mailto:phjang1983@daum.net">Jang,Pyohwan</a>
@@ -20,9 +31,38 @@
 @Configuration
 public class JestConfig {
 
+	@Value("${elasticsearch.cluster.name}")
+	private String elasticsearchClusterName;
+
+	@Value("${elasticsearch.index.name}")
+	private String elasticsearchIndexName;
+
 	@Value("${elasticsearch.host.name}")
 	private String elasticsearchHostName;
-
+
+	@Bean
+	public Client client() {
+
+		Settings settings = Settings.settingsBuilder()
+				.put("cluster.name", elasticsearchClusterName)
+				.build();
+
+		TransportClient client = TransportClient.builder()
+				.settings(settings)
+				.build();
+
+		Stream.of(StringUtils.split(elasticsearchHostName, ","))
+				.forEach(hostName -> {
+					try {
+						client.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(new URL(hostName).getHost()), 9300));
+					} catch (MalformedURLException | UnknownHostException e) {
+						e.printStackTrace();
+					}
+				});
+
+		return client;
+	}
+
 	@Bean
 	public JestClient jestClient() {
 		String[] result = elasticsearchHostName.split(",");

jakduk-core/src/main/java/com/jakduk/core/dao/JakdukDAO.java

@@ -11,6 +11,7 @@
 import com.jakduk.core.model.simple.BoardFreeOnRSS;
 import com.jakduk.core.model.simple.GalleryOnList;
 import com.jakduk.core.model.simple.UserOnHome;
+import org.apache.commons.lang3.StringUtils;
 import org.bson.types.ObjectId;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.domain.Sort;
@@ -26,6 +27,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.stream.Collectors;
 
 /**
@@ -213,7 +215,12 @@ public HomeDescription getHomeDescription() {
 
 		return homeDescription;
 	}
-
+
+	/**
+	 * 엘라스틱서치에 저장할 BoardFree 목록 가져오기.
+	 * @param commentId
+	 * @return
+	 */
 	public List<BoardFreeOnES> getBoardFreeOnES(ObjectId commentId) {
 		AggregationOperation match1 = Aggregation.match(Criteria.where("status.delete").ne(CommonConst.BOARD_HISTORY_TYPE.DELETE.name()));
 		AggregationOperation match2 = Aggregation.match(Criteria.where("_id").gt(commentId));
@@ -231,20 +238,25 @@ public List<BoardFreeOnES> getBoardFreeOnES(ObjectId commentId) {
 		AggregationResults<BoardFreeOnES> results = mongoTemplate.aggregate(aggregation, "boardFree", BoardFreeOnES.class);
 
 		List<BoardFreeOnES> posts = results.getMappedResults();
-
-		for (BoardFreeOnES post : posts) {
-			post.setSubject(post.getSubject()
-					.replaceAll("<(/)?([a-zA-Z0-9]*)(\\s[a-zA-Z0-9]*=[^>]*)?(\\s)*(/)?>","")
-					.replaceAll("\r|\n|&nbsp;",""));
 
-			post.setContent(post.getContent()
-					.replaceAll("<(/)?([a-zA-Z0-9]*)(\\s[a-zA-Z0-9]*=[^>]*)?(\\s)*(/)?>","")
-					.replaceAll("\r|\n|&nbsp;",""));
-		}
-
+		posts.forEach(post -> {
+			String subject = Optional.ofNullable(post.getSubject()).orElse("");
+			subject = StringUtils.replacePattern(subject, CommonConst.REGEX_FIND_HTML_TAG, "");
+			post.setSubject(StringUtils.replacePattern(subject, CommonConst.REGEX_FIND_HTML_WHITESPACE, ""));
+
+			String content = Optional.ofNullable(post.getContent()).orElse("");
+			content = StringUtils.replacePattern(content, CommonConst.REGEX_FIND_HTML_TAG, "");
+			post.setContent(StringUtils.replacePattern(content, CommonConst.REGEX_FIND_HTML_WHITESPACE, ""));
+		});
+
 		return posts;
 	}
-
+
+	/**
+	 * 엘라스틱서치에 저장할 BoardFreeComment 목록 가져오기.
+	 * @param commentId
+	 * @return
+	 */
 	public List<CommentOnES> getCommentOnES(ObjectId commentId) {
 		AggregationOperation match1 = Aggregation.match(Criteria.where("_id").gt(commentId));
 		AggregationOperation sort = Aggregation.sort(Direction.ASC, "_id");
@@ -261,17 +273,21 @@ public List<CommentOnES> getCommentOnES(ObjectId commentId) {
 		AggregationResults<CommentOnES> results = mongoTemplate.aggregate(aggregation, "boardFreeComment", CommentOnES.class);
 
 		List<CommentOnES> comments = results.getMappedResults();
-
-		for (CommentOnES comment : comments) {
 
-			comment.setContent(comment.getContent()
-					.replaceAll("<(/)?([a-zA-Z0-9]*)(\\s[a-zA-Z0-9]*=[^>]*)?(\\s)*(/)?>","")
-					.replaceAll("\r|\n|&nbsp;",""));
-		}
+		comments.forEach(comment -> {
+			String content = Optional.ofNullable(comment.getContent()).orElse("");
+			content = StringUtils.replacePattern(content, CommonConst.REGEX_FIND_HTML_TAG, "");
+			comment.setContent(StringUtils.replacePattern(content, CommonConst.REGEX_FIND_HTML_WHITESPACE, ""));
+		});
 
 		return comments;
 	}
-
+
+	/**
+	 * 엘라스틱서치에 저장할 Gallery 목록 가져오기.
+	 * @param commentId
+	 * @return
+	 */
 	public List<GalleryOnES> getGalleryOnES(ObjectId commentId) {
 		AggregationOperation match1 = Aggregation.match(Criteria.where("_id").gt(commentId));
 		AggregationOperation sort = Aggregation.sort(Direction.ASC, "_id");

jakduk-core/src/main/java/com/jakduk/core/exception/ServiceError.java

@@ -27,9 +27,9 @@ public enum ServiceError {
     UNAUTHORIZED_ACCESS(HttpStatus.UNAUTHORIZED, "UNAUTHORIZED_ACCESS", "common.exception.access.denied"),
     FORBIDDEN(HttpStatus.FORBIDDEN, "FORBIDDEN", "common.exception.forbidden"),
 
-
     SEND_EMAIL_FAILED(HttpStatus.INTERNAL_SERVER_ERROR, "SEND_EMAIL_FAILED", "common.exception.send.email.failed"),
-    GALLERY_IO_ERROR(HttpStatus.INTERNAL_SERVER_ERROR, "GALLERY_IO_ERROR", "common.gallery.exception.io");
+    GALLERY_IO_ERROR(HttpStatus.INTERNAL_SERVER_ERROR, "GALLERY_IO_ERROR", "common.gallery.exception.io"),
+    ELASTICSEARCH_NOT_FOUND_INDEX(HttpStatus.INTERNAL_SERVER_ERROR, "ELASTICSEARCH_NOT_FOUND_INDEX", "common.exception.elasticsearch.not.found.index");
 
     private final HttpStatus httpStatus;
     private final String code;