From 789dd87e151b25206a3d3aba97b835814c7243e7 Mon Sep 17 00:00:00 2001 From: bowenlan-amzn <bowenlan@amazon.com> Date: Tue, 17 Nov 2020 17:08:39 -0800 Subject: [PATCH 01/21] start index template API add filter by lastupdatetime, sort by priority logic --- .../indexmanagement/IndexManagementPlugin.kt | 54 +++- .../ISMTemplateService.kt | 235 ++++++++++++++++++ .../ManagedIndexCoordinator.kt | 42 +++- .../indexstatemanagement/model/ISMTemplate.kt | 130 ++++++++++ .../model/ISMTemplateMetadata.kt | 137 ++++++++++ .../indexstatemanagement/model/Policy.kt | 3 + .../resthandler/RestAddISMTemplateAction.kt | 67 +++++ .../RestDeleteISMTemplateAction.kt | 54 ++++ .../resthandler/RestGetISMTemplateAction.kt | 55 ++++ .../delete/DeleteISMTemplateAction.kt | 26 ++ .../delete/DeleteISMTemplateRequest.kt | 45 ++++ .../TransportDeleteISMTemplateAction.kt | 71 ++++++ .../ismtemplate/get/GetISMTemplateAction.kt | 25 ++ .../ismtemplate/get/GetISMTemplateRequest.kt | 44 ++++ .../ismtemplate/get/GetISMTemplateResponse.kt | 50 ++++ .../get/TransportGetISMTemplateAction.kt | 79 ++++++ .../ismtemplate/put/PutISMTemplateAction.kt | 26 ++ .../ismtemplate/put/PutISMTemplateRequest.kt | 52 ++++ .../put/TransportPutISMTemplateAction.kt | 76 ++++++ .../util/RestHandlerUtils.kt | 21 ++ .../resthandler/ISMTemplateRestAPIIT.kt | 5 + 21 files changed, 1293 insertions(+), 4 deletions(-) create mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt create mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplate.kt create mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateMetadata.kt create mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestAddISMTemplateAction.kt create mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestDeleteISMTemplateAction.kt create mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestGetISMTemplateAction.kt create mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateAction.kt create mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequest.kt create mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/TransportDeleteISMTemplateAction.kt create mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateAction.kt create mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequest.kt create mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponse.kt create mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/TransportGetISMTemplateAction.kt create mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateAction.kt create mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateRequest.kt create mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/TransportPutISMTemplateAction.kt create mode 100644 src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/IndexManagementPlugin.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/IndexManagementPlugin.kt index ae31e6055..859799346 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/IndexManagementPlugin.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/IndexManagementPlugin.kt @@ -18,14 +18,18 @@ package com.amazon.opendistroforelasticsearch.indexmanagement import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.IndexStateManagementHistory import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ManagedIndexCoordinator import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ManagedIndexRunner +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplateMetadata import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.updateindexmetadata.TransportUpdateManagedIndexMetaDataAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.updateindexmetadata.UpdateManagedIndexMetaDataAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.Policy +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestAddISMTemplateAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestAddPolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestChangePolicyAction +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestDeleteISMTemplateAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestDeletePolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestExplainAction +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestGetISMTemplateAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestGetPolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestIndexPolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestRemovePolicyAction @@ -43,6 +47,12 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagemen import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.getpolicy.TransportGetPolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.indexpolicy.IndexPolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.indexpolicy.TransportIndexPolicyAction +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.delete.DeleteISMTemplateAction +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.delete.TransportDeleteISMTemplateAction +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get.GetISMTemplateAction +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get.TransportGetISMTemplateAction +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put.PutISMTemplateAction +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put.TransportPutISMTemplateAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.removepolicy.RemovePolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.removepolicy.TransportRemovePolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.retryfailedmanagedindex.RetryFailedManagedIndexAction @@ -90,15 +100,19 @@ import org.elasticsearch.action.ActionRequest import org.elasticsearch.action.ActionResponse import org.elasticsearch.action.support.ActionFilter import org.elasticsearch.client.Client +import org.elasticsearch.cluster.NamedDiff import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver +import org.elasticsearch.cluster.metadata.Metadata import org.elasticsearch.cluster.node.DiscoveryNodes import org.elasticsearch.cluster.service.ClusterService import org.elasticsearch.common.io.stream.NamedWriteableRegistry +import org.elasticsearch.common.io.stream.Writeable import org.elasticsearch.common.settings.ClusterSettings import org.elasticsearch.common.settings.IndexScopedSettings import org.elasticsearch.common.settings.Setting import org.elasticsearch.common.settings.Settings import org.elasticsearch.common.settings.SettingsFilter +import org.elasticsearch.common.xcontent.ContextParser import org.elasticsearch.common.util.concurrent.ThreadContext import org.elasticsearch.common.xcontent.NamedXContentRegistry import org.elasticsearch.common.xcontent.XContentParser.Token @@ -132,6 +146,7 @@ internal class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, Act const val ROLLUP_BASE_URI = "$OPEN_DISTRO_BASE_URI/_rollup" const val POLICY_BASE_URI = "$ISM_BASE_URI/policies" const val ROLLUP_JOBS_BASE_URI = "$ROLLUP_BASE_URI/jobs" + const val ISM_TEMPLATE_BASE_URI = "$ISM_BASE_URI/templates" const val INDEX_MANAGEMENT_INDEX = ".opendistro-ism-config" const val INDEX_MANAGEMENT_JOB_TYPE = "opendistro-index-management" const val INDEX_STATE_MANAGEMENT_HISTORY_TYPE = "managed_index_meta_data" @@ -197,7 +212,10 @@ internal class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, Act RestIndexRollupAction(), RestStartRollupAction(), RestStopRollupAction(), - RestExplainRollupAction() + RestExplainRollupAction(), + RestAddISMTemplateAction(), + RestGetISMTemplateAction(), + RestDeleteISMTemplateAction() ) } @@ -298,10 +316,42 @@ internal class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, Act ActionPlugin.ActionHandler(StartRollupAction.INSTANCE, TransportStartRollupAction::class.java), ActionPlugin.ActionHandler(StopRollupAction.INSTANCE, TransportStopRollupAction::class.java), ActionPlugin.ActionHandler(ExplainRollupAction.INSTANCE, TransportExplainRollupAction::class.java), - ActionPlugin.ActionHandler(UpdateRollupMappingAction.INSTANCE, TransportUpdateRollupMappingAction::class.java) + ActionPlugin.ActionHandler(UpdateRollupMappingAction.INSTANCE, TransportUpdateRollupMappingAction::class.java), + ActionPlugin.ActionHandler(PutISMTemplateAction.INSTANCE, TransportPutISMTemplateAction::class.java), + ActionPlugin.ActionHandler(GetISMTemplateAction.INSTANCE, TransportGetISMTemplateAction::class.java), + ActionPlugin.ActionHandler(DeleteISMTemplateAction.INSTANCE, TransportDeleteISMTemplateAction::class.java) ) } + // override fun getNamedXContent(): MutableList<NamedXContentRegistry.Entry> { + // val entries = mutableListOf<NamedXContentRegistry.Entry>() + // val ismTemplateEntry = NamedXContentRegistry.Entry( + // Metadata.Custom::class.java, + // ISMTemplateMetadata.ISM_TEMPLATE, + // ContextParser{ p, _ -> ISMTemplateMetadata.parse(p) } + // ) + // entries.add(ismTemplateEntry) + // return entries + // } + + override fun getNamedWriteables(): MutableList<NamedWriteableRegistry.Entry> { + // ClusterModule 139 + val entries = mutableListOf<NamedWriteableRegistry.Entry>() + val ismTemplateEntry = NamedWriteableRegistry.Entry( + Metadata.Custom::class.java, + ISMTemplateMetadata.TYPE, + Writeable.Reader{ sin -> ISMTemplateMetadata(sin) } + ) + val ismTemplateEntry2 = NamedWriteableRegistry.Entry( + NamedDiff::class.java, + ISMTemplateMetadata.TYPE, + Writeable.Reader{ sin -> ISMTemplateMetadata.readDiffFrom(sin) } + ) + entries.add(ismTemplateEntry) + entries.add(ismTemplateEntry2) + return entries + } + override fun getTransportInterceptors(namedWriteableRegistry: NamedWriteableRegistry, threadContext: ThreadContext): List<TransportInterceptor> { return listOf(rollupInterceptor) } diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt new file mode 100644 index 000000000..187d222bb --- /dev/null +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt @@ -0,0 +1,235 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement + +import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.ismTemplates +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.putISMTemplate +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.removeISMTemplate +import org.apache.logging.log4j.LogManager +import org.apache.lucene.util.automaton.Operations +import org.elasticsearch.action.ActionListener +import org.elasticsearch.action.support.master.AcknowledgedResponse +import org.elasticsearch.cluster.ClusterState +import org.elasticsearch.cluster.ClusterStateUpdateTask +import org.elasticsearch.cluster.metadata.IndexMetadata +import org.elasticsearch.cluster.metadata.Metadata +import org.elasticsearch.cluster.service.ClusterService +import org.elasticsearch.common.Priority +import org.elasticsearch.common.Strings +import org.elasticsearch.common.ValidationException +import org.elasticsearch.common.inject.Inject +import org.elasticsearch.common.regex.Regex +import org.elasticsearch.common.unit.TimeValue +import org.elasticsearch.indices.InvalidIndexTemplateException +import java.util.* +import java.util.stream.Collectors + +private val log = LogManager.getLogger(ISMTemplateService::class.java) + +// MetadataIndexTemplateService +class ISMTemplateService @Inject constructor( + val clusterService: ClusterService +) { + /** + * save ISM template to cluster state metadata + */ + fun putISMTemplate(templateName: String, template: ISMTemplate, masterTimeout: TimeValue, + listener: ActionListener<AcknowledgedResponse>) { + clusterService.submitStateUpdateTask( + IndexManagementPlugin.PLUGIN_NAME, + object : ClusterStateUpdateTask(Priority.NORMAL) { + override fun execute(currentState: ClusterState): ClusterState { + return addISMTemplate(currentState, templateName, template) + } + + override fun onFailure(source: String, e: Exception) { + listener.onFailure(e) + } + + override fun timeout(): TimeValue = masterTimeout + + override fun clusterStateProcessed(source: String, oldState: ClusterState, newState: ClusterState) { + listener.onResponse(AcknowledgedResponse(true)) + } + } + ) + } + + fun addISMTemplate(currentState: ClusterState, templateName: String, template: ISMTemplate): ClusterState { + val existingTemplates = currentState.metadata.ismTemplates() + val existingTemplate = existingTemplates[templateName] + + log.info("existing matching template $existingTemplate") + log.info("input template $template") + + if (template == existingTemplate) return currentState + + // find templates with overlapping index pattern + val overlaps = findConflictingISMTemplates(templateName, template.indexPatterns, template.priority, existingTemplates) + log.info("find overlapping templates $overlaps") + if (overlaps.isNotEmpty()) { + val esg = "new ism template $templateName has index pattern ${template.indexPatterns} matching existing templates ${overlaps.entries.stream().map { "${it.key} => ${it.value}" }.collect(Collectors.joining(","))}, please use a different priority than ${template.priority}" + throw IllegalArgumentException(esg) + } + + validateFormat(templateName, template.indexPatterns) + + log.info("updating ISM template $templateName") + return ClusterState.builder(currentState).metadata(Metadata.builder(currentState.metadata()) + .putISMTemplate(templateName, template, existingTemplates)).build() + } + + /** + * remove ISM template from cluster state metadata + */ + fun deleteISMTemplate(templateName: String, masterTimeout: TimeValue, listener: ActionListener<AcknowledgedResponse>) { + log.info("service remove template") + clusterService.submitStateUpdateTask( + IndexManagementPlugin.PLUGIN_NAME, + object : ClusterStateUpdateTask(Priority.NORMAL) { + override fun execute(currentState: ClusterState): ClusterState { + log.info("service remove template $templateName") + val existingTemplates = currentState.metadata.ismTemplates() + return ClusterState.builder(currentState).metadata(Metadata.builder(currentState.metadata).removeISMTemplate(templateName, existingTemplates)).build() + } + + override fun onFailure(source: String, e: Exception) { + listener.onFailure(e) + } + + override fun timeout(): TimeValue = masterTimeout + + override fun clusterStateProcessed(source: String, oldState: ClusterState, newState: ClusterState) { + listener.onResponse(AcknowledgedResponse(true)) + } + } + ) + } + + companion object { + /** + * find the matching template name for the given index name + * + * filter out hidden index + * filter out older index than template lastUpdateTime + */ + // findV2Template + fun findMatchingISMTemplate(ismTemplates: Map<String, ISMTemplate>, indexMetadata: IndexMetadata): String? { + val indexName = indexMetadata.index.name + + // don't include hidden index + val isHidden = IndexMetadata.INDEX_HIDDEN_SETTING.get(indexMetadata.settings) + log.info("index $indexName is hidden $isHidden") + if (isHidden) return null + + val ismTemplates = ismTemplates.filter { (_, template) -> + log.info("template last update time: ${template.lastUpdatedTime.toEpochMilli()}") + log.info("index create time: ${indexMetadata.creationDate}") + log.info("is template older? ${template.lastUpdatedTime.toEpochMilli() < indexMetadata.creationDate}") + template.lastUpdatedTime.toEpochMilli() < indexMetadata.creationDate + } + + // traverse all ism templates for matching ones + val patternMatchPredicate = { pattern: String -> Regex.simpleMatch(pattern, indexName) } + val matchedTemplates = mutableMapOf<ISMTemplate, String>() + ismTemplates.forEach { (templateName, template) -> + val matched = template.indexPatterns.stream().anyMatch(patternMatchPredicate) + if (matched) matchedTemplates[template] = templateName + } + + if (matchedTemplates.isEmpty()) return null + log.info("all matching templates $matchedTemplates") + + // sort by template priority + val winner = matchedTemplates.keys.maxBy { it.priority } + log.info("winner with highest priority is $winner") + return matchedTemplates[winner] + } + + fun validateFormat(templateName: String, indexPatterns: List<String>) { + val validationErrors = mutableListOf<String>() + if (templateName.contains(" ")) { + validationErrors.add("name must not contain a space") + } + if (templateName.contains(",")) { + validationErrors.add("name must not contain a ','") + } + if (templateName.contains("#")) { + validationErrors.add("name must not contain a '#'") + } + if (templateName.contains("*")) { + validationErrors.add("name must not contain a '*'") + } + if (templateName.startsWith("_")) { + validationErrors.add("name must not start with '_'") + } + if (templateName.toLowerCase(Locale.ROOT) != templateName) { + validationErrors.add("name must be lower cased") + } + for (indexPattern in indexPatterns) { + if (indexPattern.contains(" ")) { + validationErrors.add("index_patterns [$indexPattern] must not contain a space") + } + if (indexPattern.contains(",")) { + validationErrors.add("index_pattern [$indexPattern] must not contain a ','") + } + if (indexPattern.contains("#")) { + validationErrors.add("index_pattern [$indexPattern] must not contain a '#'") + } + if (indexPattern.contains(":")) { + validationErrors.add("index_pattern [$indexPattern] must not contain a ':'") + } + if (indexPattern.startsWith("_")) { + validationErrors.add("index_pattern [$indexPattern] must not start with '_'") + } + if (!Strings.validFileNameExcludingAstrix(indexPattern)) { + validationErrors.add("index_pattern [" + indexPattern + "] must not contain the following characters " + + Strings.INVALID_FILENAME_CHARS) + } + } + + if (validationErrors.size > 0) { + val validationException = ValidationException() + validationException.addValidationErrors(validationErrors) + throw InvalidIndexTemplateException(templateName, validationException.message) + } + } + + /** + * find templates whose index patterns overlap with given template + * + * @return map of overlapping template name to its index patterns + */ + // addIndexTemplateV2 findConflictingV2Templates + fun findConflictingISMTemplates(candidate: String, indexPatterns: List<String>, priority: Int, ismTemplates: Map<String, ISMTemplate>): Map<String, List<String>> { + // focus on template with same priority + val ismTemplates = ismTemplates.filter { it.value.priority == priority } + val automaton1 = Regex.simpleMatchToAutomaton(*indexPatterns.toTypedArray()) + val overlappingTemplates = mutableMapOf<String, List<String>>() + ismTemplates.forEach { (templateName, template) -> + val automaton2 = Regex.simpleMatchToAutomaton(*template.indexPatterns.toTypedArray()) + if (!Operations.isEmpty(Operations.intersection(automaton1, automaton2))) { + log.info("existing template $templateName overlaps candidate $candidate") + overlappingTemplates[templateName] = template.indexPatterns + } + } + overlappingTemplates.remove(candidate) + return overlappingTemplates + } + } +} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt index 9b4be1412..54a8f0de0 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt @@ -18,14 +18,15 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanageme import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementIndices import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.parseWithType +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ISMTemplateService.Companion.findMatchingISMTemplate import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.getClusterStateManagedIndexConfig import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.getManagedIndexMetaData import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.getPolicyID import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.retry +import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.suspendUntil import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.shouldCreateManagedIndexConfig import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.shouldDeleteManagedIndexConfig import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.shouldDeleteManagedIndexMetaData -import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.suspendUntil import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.coordinator.ClusterStateManagedIndexConfig import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData @@ -46,6 +47,7 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagemen import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.getSweptManagedIndexSearchRequest import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.isFailed import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.isPolicyCompleted +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.ismTemplates import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.updateEnableManagedIndexRequest import com.amazon.opendistroforelasticsearch.indexmanagement.util.NO_ID import kotlinx.coroutines.CoroutineName @@ -261,10 +263,38 @@ class ManagedIndexCoordinator( if (it.value.shouldDeleteManagedIndexMetaData()) indicesToRemoveManagedIndexMetaDataFrom.add(it.value.index) } - updateManagedIndices(updateManagedIndicesRequests + indicesDeletedRequests, hasCreateRequests) + // check newly created indices matching any ISM templates + val updateMatchingIndexReqs = getMatchingIndicesUpdateReqs(event.state(), event.indicesCreated()) + if (updateMatchingIndexReqs.isNotEmpty()) hasCreateRequests = true + + updateManagedIndices(updateManagedIndicesRequests + updateMatchingIndexReqs + indicesDeletedRequests, hasCreateRequests) clearManagedIndexMetaData(indicesToRemoveManagedIndexMetaDataFrom) } + /** + * Get update requests for indices matching any ISM templates + */ + fun getMatchingIndicesUpdateReqs(clusterState: ClusterState, indexNames: List<String>): List<DocWriteRequest<*>> { + val indexMetadatas = clusterState.metadata.indices + val templates = clusterState.metadata.ismTemplates() + + val matchingTemplates = indexNames.map { indexName -> + indexName to findMatchingISMTemplate(templates, indexMetadatas[indexName]) + }.toMap() + + val updateManagedIndexReqs = mutableListOf<DocWriteRequest<*>>() + matchingTemplates.filter { (_, template) -> template != null }.forEach { (index, template) -> + val indexUuids = indexMetadatas[index].indexUUID + val policyID = templates[template]?.policyID + if (indexUuids != null && policyID != null) { + logger.info("create request for index $index matching template $template") + updateManagedIndexReqs.add(managedIndexConfigIndexRequest(index, indexUuids, policyID, jobInterval)) + } + } + + return updateManagedIndexReqs + } + /** * Background sweep process that periodically sweeps for updates to ManagedIndices * @@ -317,6 +347,13 @@ class ManagedIndexCoordinator( @OpenForTesting suspend fun sweep() { val currentManagedIndices = sweepManagedIndexJobs(client, ismIndices.indexManagementIndexExists()) + + // check all un-managed indices, if its name matches any template and older than that template + val unManagedIndices = clusterService.state().metadata.indices.values().filterNotNull() + .filter { it.value.indexUUID !in currentManagedIndices.keys }.map { it.value.index.name } + val updateMatchingIndicesReqs = getMatchingIndicesUpdateReqs(clusterService.state(), unManagedIndices) + updateManagedIndices(updateMatchingIndicesReqs, updateMatchingIndicesReqs.isNotEmpty()) + val clusterStateManagedIndices = sweepClusterState(clusterService.state()) val createManagedIndexRequests = @@ -330,6 +367,7 @@ class ManagedIndexCoordinator( val requests = createManagedIndexRequests + deleteManagedIndexRequests updateManagedIndices(requests, createManagedIndexRequests.isNotEmpty()) clearManagedIndexMetaData(indicesToDeleteManagedIndexMetaDataFrom) + lastFullSweepTimeNano = System.nanoTime() } diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplate.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplate.kt new file mode 100644 index 000000000..2ea7e32ae --- /dev/null +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplate.kt @@ -0,0 +1,130 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model + +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.instant +import org.apache.logging.log4j.LogManager +import org.elasticsearch.cluster.AbstractDiffable +import org.elasticsearch.cluster.Diff +import org.elasticsearch.common.io.stream.StreamInput +import org.elasticsearch.common.io.stream.StreamOutput +import org.elasticsearch.common.xcontent.ToXContent +import org.elasticsearch.common.xcontent.ToXContentObject +import org.elasticsearch.common.xcontent.XContentBuilder +import org.elasticsearch.common.xcontent.XContentParser +import org.elasticsearch.common.xcontent.XContentParser.Token +import org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken +import java.io.IOException +import java.lang.IllegalArgumentException +import java.time.Instant + +private val log = LogManager.getLogger(ISMTemplate::class.java) + +// ComposableIndexTemplate +// ManagedIndexMetaData +data class ISMTemplate( + val indexPatterns: List<String>, + val policyID: String, + val priority: Int, + val lastUpdatedTime: Instant +) : ToXContentObject, AbstractDiffable<ISMTemplate>() { + + init { + require(indexPatterns.isNotEmpty()) { "at least give one index pattern" } + } + + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + return builder.startObject() + .field(INDEX_PATTERN, indexPatterns) + .field(POLICY_ID, policyID) + .field(PRIORITY, priority) + .field(LAST_UPDATED_TIME_FIELD, lastUpdatedTime) + .endObject() + } + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + sin.readStringList(), + sin.readString(), + sin.readInt(), + sin.readInstant() + ) + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeStringCollection(indexPatterns) + out.writeString(policyID) + out.writeInt(priority) + out.writeInstant(lastUpdatedTime) + } + + companion object { + const val INDEX_PATTERN = "index_patterns" + const val POLICY_ID = "policy_id" + const val PRIORITY = "priority" + const val LAST_UPDATED_TIME_FIELD = "last_updated_time" + + fun parse(xcp: XContentParser): ISMTemplate { + val indexPatterns: MutableList<String> = mutableListOf() + var policyID: String? = null + var priority = 0 + var lastUpdatedTime: Instant? = null + + log.info("current token ${xcp.currentToken()}") + ensureExpectedToken(Token.START_OBJECT, xcp.nextToken(), xcp::getTokenLocation) + while (xcp.nextToken() != Token.END_OBJECT) { + val fieldName = xcp.currentName() + log.info("parse field name $fieldName") + xcp.nextToken() + + when (fieldName) { + INDEX_PATTERN -> { + ensureExpectedToken(Token.START_ARRAY, xcp.currentToken(), xcp::getTokenLocation) + while (xcp.nextToken() != Token.END_ARRAY) { + indexPatterns.add(xcp.text()) + log.info("field $indexPatterns") + } + } + POLICY_ID -> { + policyID = xcp.text() + log.info("field $policyID") + } + PRIORITY -> { + priority = if (xcp.currentToken() == Token.VALUE_NULL) 0 else xcp.intValue() + } + LAST_UPDATED_TIME_FIELD -> { + lastUpdatedTime = xcp.instant() + log.info("field last update time $lastUpdatedTime") + } + else -> throw IllegalArgumentException("Invalid field: [$fieldName] found in ISMTemplate.") + } + } + + val result = ISMTemplate( + indexPatterns, + requireNotNull(policyID) { "policy id is null" }, + priority, + lastUpdatedTime ?: Instant.now() + ) + + log.info("ism template parse result $result") + // TODO check index pattern is empty or not + return result + } + + fun readISMTemplateDiffFrom(sin: StreamInput): Diff<ISMTemplate> = AbstractDiffable.readDiffFrom(::ISMTemplate, sin) + } +} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateMetadata.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateMetadata.kt new file mode 100644 index 000000000..4082f599f --- /dev/null +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateMetadata.kt @@ -0,0 +1,137 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model + +import org.apache.logging.log4j.LogManager +import org.elasticsearch.Version +import org.elasticsearch.cluster.AbstractNamedDiffable +import org.elasticsearch.cluster.Diff +import org.elasticsearch.cluster.DiffableUtils +import org.elasticsearch.cluster.NamedDiff +import org.elasticsearch.cluster.metadata.ComponentTemplate +import org.elasticsearch.cluster.metadata.Metadata +import org.elasticsearch.common.ParseField +import org.elasticsearch.common.io.stream.StreamInput +import org.elasticsearch.common.io.stream.StreamOutput +import org.elasticsearch.common.xcontent.ConstructingObjectParser +import org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg +import org.elasticsearch.common.xcontent.ContextParser +import org.elasticsearch.common.xcontent.ToXContent +import org.elasticsearch.common.xcontent.XContentBuilder +import org.elasticsearch.common.xcontent.XContentParser +import org.elasticsearch.common.xcontent.XContentParserUtils +import org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken +import java.util.* +import java.util.stream.Stream + +private val log = LogManager.getLogger(ISMTemplateMetadata::class.java) + +/** + * <template_name>: ISMTemplate + */ +// ComponentTemplateMetadata +// EnrichMetadata +class ISMTemplateMetadata(val ismTemplates: Map<String, ISMTemplate>): Metadata.Custom { + + constructor(sin: StreamInput) : this( + sin.readMap(StreamInput::readString, ::ISMTemplate) + ) + + override fun writeTo(out: StreamOutput) { + out.writeMap(ismTemplates, StreamOutput::writeString) { stream, `val` -> `val`.writeTo(stream) } + } + + override fun diff(before: Metadata.Custom): Diff<Metadata.Custom> { + return ISMTemplateMetadataDiff((before as ISMTemplateMetadata), this) + } + + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + log.info("ism template metadata toXContent: $ismTemplates") + builder.startObject(ISM_TEMPLATE.preferredName) + ismTemplates.forEach { (k, v) -> + builder.field(k, v) + } + builder.endObject() + return builder + } + + override fun getWriteableName(): String = TYPE + + override fun getMinimalSupportedVersion(): Version = Version.V_7_7_0 + + override fun context(): EnumSet<Metadata.XContentContext> = Metadata.ALL_CONTEXTS + + class ISMTemplateMetadataDiff: NamedDiff<Metadata.Custom> { + + val ismTemplateDiff: Diff<Map<String, ISMTemplate>> + + constructor(before: ISMTemplateMetadata, after: ISMTemplateMetadata) { + this.ismTemplateDiff = DiffableUtils.diff(before.ismTemplates, after.ismTemplates, DiffableUtils.getStringKeySerializer()) + } + + constructor(sin: StreamInput) { + this.ismTemplateDiff = DiffableUtils.readJdkMapDiff(sin, DiffableUtils.getStringKeySerializer(), + ::ISMTemplate, ISMTemplate.Companion::readISMTemplateDiffFrom) + } + + override fun writeTo(out: StreamOutput) { + ismTemplateDiff.writeTo(out) + } + + override fun getWriteableName(): String = TYPE + + override fun apply(part: Metadata.Custom): Metadata.Custom { + return ISMTemplateMetadata(ismTemplateDiff.apply((part as ISMTemplateMetadata).ismTemplates)) + } + + } + + companion object { + val TYPE = "ism_template" + val ISM_TEMPLATE = ParseField("ism_template") + + val PARSER = ConstructingObjectParser<ISMTemplateMetadata, Void>(TYPE, false + ) { a -> ISMTemplateMetadata(a[0] as Map<String, ISMTemplate>) } + + init { + PARSER.declareObject(constructorArg(), ContextParser<Void, Map<String, ISMTemplate>> { p, _ -> + val templates = mutableMapOf<String, ISMTemplate>() + while (p.nextToken() != XContentParser.Token.END_OBJECT) { + val name = p.currentName() + templates[name] = ISMTemplate.parse(p) + } + templates + }, ISM_TEMPLATE) + } + + fun fromStreamInput(sin: StreamInput) = ISMTemplateMetadata(sin.readMap(StreamInput::readString, ::ISMTemplate)) + + // fun parse(xcp: XContentParser): ISMTemplateMetadata = PARSER.parse(xcp, null) + fun parse(xcp: XContentParser): ISMTemplateMetadata { + val ismTemplates = mutableMapOf<String, ISMTemplate>() + log.info("ism template metadata parse, first token ${xcp.currentToken()}") + ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp::getTokenLocation) + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + val fieldName = xcp.currentName() + log.info("current field name $fieldName") + ismTemplates[fieldName] = ISMTemplate.parse(xcp) + } + return ISMTemplateMetadata(ismTemplates) + } + + fun readDiffFrom(sin: StreamInput): NamedDiff<Metadata.Custom> = ISMTemplateMetadataDiff(sin) + } +} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/Policy.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/Policy.kt index 10fe89fc8..69b2e79aa 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/Policy.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/Policy.kt @@ -19,6 +19,7 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.instant import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.optionalTimeField import com.amazon.opendistroforelasticsearch.indexmanagement.util.IndexUtils import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.WITH_TYPE +import org.apache.logging.log4j.LogManager import org.elasticsearch.common.io.stream.StreamInput import org.elasticsearch.common.io.stream.StreamOutput import org.elasticsearch.common.io.stream.Writeable @@ -32,6 +33,8 @@ import org.elasticsearch.index.seqno.SequenceNumbers import java.io.IOException import java.time.Instant +private val log = LogManager.getLogger(Policy::class.java) + data class Policy( val id: String = NO_ID, val seqNo: Long = SequenceNumbers.UNASSIGNED_SEQ_NO, diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestAddISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestAddISMTemplateAction.kt new file mode 100644 index 000000000..18972b254 --- /dev/null +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestAddISMTemplateAction.kt @@ -0,0 +1,67 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler + +import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin.Companion.ISM_TEMPLATE_BASE_URI +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put.PutISMTemplateAction +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put.PutISMTemplateRequest +import org.apache.logging.log4j.LogManager +import org.elasticsearch.action.support.master.MasterNodeRequest.DEFAULT_MASTER_NODE_TIMEOUT +import org.elasticsearch.client.node.NodeClient +import org.elasticsearch.common.xcontent.XContentHelper +import org.elasticsearch.rest.BaseRestHandler +import org.elasticsearch.rest.RestHandler.Route +import org.elasticsearch.rest.RestRequest +import org.elasticsearch.rest.RestRequest.Method.PUT +import org.elasticsearch.rest.action.RestToXContentListener +import java.lang.IllegalArgumentException +import java.time.Instant + +private val log = LogManager.getLogger(RestAddISMTemplateAction::class.java) + +// RestIndexPolicyAction +class RestAddISMTemplateAction : BaseRestHandler() { + override fun routes(): List<Route> { + return listOf( + Route(PUT, ISM_TEMPLATE_BASE_URI), + Route(PUT, "$ISM_TEMPLATE_BASE_URI/{templateID}") + ) + } + + override fun getName(): String = "add_ism_template_action" + + override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { + log.info("${request.method()} $ISM_TEMPLATE_BASE_URI") + + val templateName = request.param("templateID", "") + if (templateName == "") { throw IllegalArgumentException("Missing template name") } + + log.info("request content ${XContentHelper.convertToMap(request.requiredContent(), false, request.xContentType).v2()}") + + val xcp = request.contentParser() + // ISMTemplate.show(xcp) + val ismTemplate = ISMTemplate.parse(xcp).copy(lastUpdatedTime = Instant.now()) + log.info("rest template $ismTemplate") + + val masterTimeout = request.paramAsTime("master_timeout", DEFAULT_MASTER_NODE_TIMEOUT) + val addISMTemplateRequest = PutISMTemplateRequest(templateName, ismTemplate).masterNodeTimeout(masterTimeout) + + return RestChannelConsumer { channel -> + client.execute(PutISMTemplateAction.INSTANCE, addISMTemplateRequest, RestToXContentListener(channel)) + } + } +} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestDeleteISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestDeleteISMTemplateAction.kt new file mode 100644 index 000000000..6c7b84c17 --- /dev/null +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestDeleteISMTemplateAction.kt @@ -0,0 +1,54 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler + +import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin.Companion.ISM_TEMPLATE_BASE_URI +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.delete.DeleteISMTemplateAction +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.delete.DeleteISMTemplateRequest +import org.apache.logging.log4j.LogManager +import org.elasticsearch.client.node.NodeClient +import org.elasticsearch.rest.BaseRestHandler +import org.elasticsearch.rest.RestHandler +import org.elasticsearch.rest.RestHandler.Route +import org.elasticsearch.rest.RestRequest +import org.elasticsearch.rest.RestRequest.Method.DELETE +import org.elasticsearch.rest.action.RestToXContentListener +import java.lang.IllegalArgumentException + +private val log = LogManager.getLogger(RestDeleteISMTemplateAction::class.java) + +class RestDeleteISMTemplateAction : BaseRestHandler() { + override fun routes(): List<Route> { + return listOf( + Route(DELETE, "${ISM_TEMPLATE_BASE_URI}/{templateID}") + ) + } + + override fun getName(): String = "remove_ism_template_action" + + override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { + log.info("${request.method()} $ISM_TEMPLATE_BASE_URI") + + val templateName = request.param("templateID", "") + if (templateName == "") { throw IllegalArgumentException("Missing template name") } + + val deleteISMTemplateRequest = DeleteISMTemplateRequest(templateName) + + return RestChannelConsumer { channel -> + client.execute(DeleteISMTemplateAction.INSTANCE, deleteISMTemplateRequest, RestToXContentListener(channel)) + } + } +} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestGetISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestGetISMTemplateAction.kt new file mode 100644 index 000000000..53ad40610 --- /dev/null +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestGetISMTemplateAction.kt @@ -0,0 +1,55 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler + +import org.apache.logging.log4j.LogManager +import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin.Companion.ISM_TEMPLATE_BASE_URI +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get.GetISMTemplateAction +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get.GetISMTemplateRequest +import org.elasticsearch.action.support.master.MasterNodeRequest.DEFAULT_MASTER_NODE_TIMEOUT +import org.elasticsearch.client.node.NodeClient +import org.elasticsearch.common.Strings +import org.elasticsearch.rest.BaseRestHandler +import org.elasticsearch.rest.RestHandler +import org.elasticsearch.rest.RestHandler.Route +import org.elasticsearch.rest.RestRequest +import org.elasticsearch.rest.RestRequest.Method.GET +import org.elasticsearch.rest.action.RestToXContentListener + +private val log = LogManager.getLogger(RestGetISMTemplateAction::class.java) + +class RestGetISMTemplateAction : BaseRestHandler() { + override fun routes(): List<Route> { + return listOf( + Route(GET, ISM_TEMPLATE_BASE_URI), + Route(GET, "$ISM_TEMPLATE_BASE_URI/{templateID}") + ) + } + + override fun getName(): String = "get_ism_template_action" + + override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { + log.info("${request.method()} $ISM_TEMPLATE_BASE_URI") + + val templateNames = Strings.splitStringByCommaToArray(request.param("templateID")) + val masterTimeout = request.paramAsTime("master_timeout", DEFAULT_MASTER_NODE_TIMEOUT) + val getISMTemplateReq = GetISMTemplateRequest(templateNames).masterNodeTimeout(masterTimeout) + + return RestChannelConsumer { channel -> + client.execute(GetISMTemplateAction.INSTANCE, getISMTemplateReq, RestToXContentListener(channel)) + } + } +} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateAction.kt new file mode 100644 index 000000000..cd0584b7e --- /dev/null +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateAction.kt @@ -0,0 +1,26 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.delete + +import org.elasticsearch.action.ActionType +import org.elasticsearch.action.support.master.AcknowledgedResponse + +class DeleteISMTemplateAction: ActionType<AcknowledgedResponse>(NAME, ::AcknowledgedResponse) { + companion object { + val NAME = "cluster:admin/opendistro/ism/templates/remove" + val INSTANCE = DeleteISMTemplateAction() + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequest.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequest.kt new file mode 100644 index 000000000..748436224 --- /dev/null +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequest.kt @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.delete + +import org.elasticsearch.action.ActionRequestValidationException +import org.elasticsearch.action.support.master.MasterNodeRequest +import org.elasticsearch.common.io.stream.StreamInput +import org.elasticsearch.common.io.stream.StreamOutput + +class DeleteISMTemplateRequest : MasterNodeRequest<DeleteISMTemplateRequest> { + + val templateName: String + + constructor( + templateName: String + ) : super() { + this.templateName = templateName + } + + constructor(sin: StreamInput) : super(sin) { + templateName = sin.readString() + } + + override fun writeTo(out: StreamOutput) { + super.writeTo(out) + out.writeString(templateName) + } + + override fun validate(): ActionRequestValidationException? { + return null + } +} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/TransportDeleteISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/TransportDeleteISMTemplateAction.kt new file mode 100644 index 000000000..c9a0cc64f --- /dev/null +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/TransportDeleteISMTemplateAction.kt @@ -0,0 +1,71 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.delete + +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ISMTemplateService +import org.apache.logging.log4j.LogManager +import org.elasticsearch.action.ActionListener +import org.elasticsearch.action.support.ActionFilters +import org.elasticsearch.action.support.master.AcknowledgedResponse +import org.elasticsearch.action.support.master.TransportMasterNodeAction +import org.elasticsearch.client.Client +import org.elasticsearch.cluster.ClusterState +import org.elasticsearch.cluster.block.ClusterBlockException +import org.elasticsearch.cluster.block.ClusterBlockLevel +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver +import org.elasticsearch.cluster.service.ClusterService +import org.elasticsearch.common.inject.Inject +import org.elasticsearch.common.io.stream.StreamInput +import org.elasticsearch.common.io.stream.Writeable +import org.elasticsearch.threadpool.ThreadPool +import org.elasticsearch.transport.TransportService + +private val log = LogManager.getLogger(TransportDeleteISMTemplateAction::class.java) + +class TransportDeleteISMTemplateAction @Inject constructor( + transportService: TransportService, + clusterService: ClusterService, + threadPool: ThreadPool, + actionFilters: ActionFilters, + indexNameExpressionResolver: IndexNameExpressionResolver, + val client: Client, + val ismTemplateService: ISMTemplateService +) : TransportMasterNodeAction<DeleteISMTemplateRequest, AcknowledgedResponse>( + DeleteISMTemplateAction.NAME, + transportService, + clusterService, + threadPool, + actionFilters, + Writeable.Reader { DeleteISMTemplateRequest(it) }, + indexNameExpressionResolver +) { + override fun executor(): String { + return ThreadPool.Names.SAME + } + + override fun read(sin: StreamInput): AcknowledgedResponse { + return AcknowledgedResponse(sin) + } + + override fun masterOperation(request: DeleteISMTemplateRequest, state: ClusterState, listener: ActionListener<AcknowledgedResponse>) { + ismTemplateService.deleteISMTemplate(request.templateName, request.masterNodeTimeout(), listener) + } + + override fun checkBlock(request: DeleteISMTemplateRequest, state: ClusterState): ClusterBlockException? { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE) + } + +} \ No newline at end of file diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateAction.kt new file mode 100644 index 000000000..ebf148caa --- /dev/null +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateAction.kt @@ -0,0 +1,25 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get + +import org.elasticsearch.action.ActionType + +class GetISMTemplateAction : ActionType<GetISMTemplateResponse>(NAME, ::GetISMTemplateResponse) { + companion object { + val NAME = "cluster:admin/opendistro/ism/templates/read" + val INSTANCE = GetISMTemplateAction() + } +} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequest.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequest.kt new file mode 100644 index 000000000..f5c370b6a --- /dev/null +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequest.kt @@ -0,0 +1,44 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get + +import org.elasticsearch.action.ActionRequestValidationException +import org.elasticsearch.action.support.master.MasterNodeRequest +import org.elasticsearch.common.io.stream.StreamInput +import org.elasticsearch.common.io.stream.StreamOutput + +// GetIndexTemplatesResponse +class GetISMTemplateRequest : MasterNodeRequest<GetISMTemplateRequest> { + + val templateNames: Array<String> + + constructor(sin: StreamInput) : super(sin) { + templateNames = sin.readStringArray() + } + + override fun writeTo(out: StreamOutput) { + super.writeTo(out) + out.writeStringArray(templateNames) + } + + constructor(templateName: Array<String>) : super() { + this.templateNames = templateName + } + + override fun validate(): ActionRequestValidationException? { + return null + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponse.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponse.kt new file mode 100644 index 000000000..be21f7194 --- /dev/null +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponse.kt @@ -0,0 +1,50 @@ +package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get + +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate +import org.elasticsearch.action.ActionResponse +import org.elasticsearch.common.ParseField +import org.elasticsearch.common.io.stream.StreamInput +import org.elasticsearch.common.io.stream.StreamOutput +import org.elasticsearch.common.xcontent.ToXContent +import org.elasticsearch.common.xcontent.ToXContentObject +import org.elasticsearch.common.xcontent.XContentBuilder + +// GetComposableIndexTemplateAction.Response +class GetISMTemplateResponse : ActionResponse, ToXContentObject { + + val ismTemplates: Map<String, ISMTemplate> + + constructor(ismTemplates: Map<String, ISMTemplate>) : super() { + this.ismTemplates = ismTemplates + } + + constructor(sin: StreamInput) : super(sin) { + val size = sin.readVInt() + ismTemplates = mutableMapOf() + repeat(size) { + ismTemplates.put(sin.readString(), ISMTemplate(sin)) + } + } + + override fun writeTo(out: StreamOutput) { + out.writeVInt(ismTemplates.size) + ismTemplates.forEach { (k, v) -> + out.writeString(k) + v.writeTo(out) + } + } + + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + builder.startObject().startArray(ISM_TEMPLATES) + ismTemplates.forEach { (k, v) -> + builder.startObject().field(NAME, k).field(ISM_TEMPLATE, v).endObject() + } + return builder.endArray().endObject() + } + + companion object { + val ISM_TEMPLATES = "ism_templates" + val NAME = "name" + val ISM_TEMPLATE = "ism_template" + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/TransportGetISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/TransportGetISMTemplateAction.kt new file mode 100644 index 000000000..87ef0b9b2 --- /dev/null +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/TransportGetISMTemplateAction.kt @@ -0,0 +1,79 @@ +package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get + +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ISMTemplateService +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.ismTemplates +import org.apache.logging.log4j.LogManager +import org.elasticsearch.ResourceNotFoundException +import org.elasticsearch.action.ActionListener +import org.elasticsearch.action.support.ActionFilters +import org.elasticsearch.action.support.master.TransportMasterNodeAction +import org.elasticsearch.client.Client +import org.elasticsearch.cluster.ClusterState +import org.elasticsearch.cluster.block.ClusterBlockException +import org.elasticsearch.cluster.block.ClusterBlockLevel +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver +import org.elasticsearch.cluster.service.ClusterService +import org.elasticsearch.common.inject.Inject +import org.elasticsearch.common.io.stream.StreamInput +import org.elasticsearch.common.io.stream.Writeable +import org.elasticsearch.common.regex.Regex +import org.elasticsearch.threadpool.ThreadPool +import org.elasticsearch.transport.TransportService + + +private val log = LogManager.getLogger(TransportGetISMTemplateAction::class.java) + +// TransportGetComposableIndexTemplateAction +class TransportGetISMTemplateAction @Inject constructor( + transportService: TransportService, + clusterService: ClusterService, + threadPool: ThreadPool, + actionFilters: ActionFilters, + indexNameExpressionResolver: IndexNameExpressionResolver, + val client: Client, + val ismTemplateService: ISMTemplateService +) : TransportMasterNodeAction<GetISMTemplateRequest, GetISMTemplateResponse>( + GetISMTemplateAction.NAME, + transportService, + clusterService, + threadPool, + actionFilters, + Writeable.Reader { GetISMTemplateRequest(it) }, + indexNameExpressionResolver +) { + override fun executor(): String { + return ThreadPool.Names.SAME + } + + override fun read(sin: StreamInput): GetISMTemplateResponse { + return GetISMTemplateResponse(sin) + } + + override fun masterOperation(request: GetISMTemplateRequest, state: ClusterState, listener: ActionListener<GetISMTemplateResponse>) { + val allTemplates = state.metadata.ismTemplates() + if (request.templateNames.isEmpty()) { + listener.onResponse(GetISMTemplateResponse(allTemplates)) + return + } + + val results = mutableMapOf<String, ISMTemplate>() + val templateNames = request.templateNames + templateNames.forEach { name -> + allTemplates.forEach { (templateName, template) -> + when { + Regex.simpleMatch(name, templateName) -> results[templateName] = template + allTemplates.containsKey(name) -> results[templateName] = template + else -> throw ResourceNotFoundException("index template matching [$name] not found") + } + } + } + + listener.onResponse(GetISMTemplateResponse(results)) + } + + override fun checkBlock(request: GetISMTemplateRequest, state: ClusterState): ClusterBlockException? { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE) + } + +} \ No newline at end of file diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateAction.kt new file mode 100644 index 000000000..25bf14320 --- /dev/null +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateAction.kt @@ -0,0 +1,26 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put + +import org.elasticsearch.action.ActionType +import org.elasticsearch.action.support.master.AcknowledgedResponse + +class PutISMTemplateAction : ActionType<AcknowledgedResponse>(NAME, ::AcknowledgedResponse) { + companion object { + val NAME = "cluster:admin/opendistro/ism/templates/add" + val INSTANCE = PutISMTemplateAction() + } +} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateRequest.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateRequest.kt new file mode 100644 index 000000000..53b33866a --- /dev/null +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateRequest.kt @@ -0,0 +1,52 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put + +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate +import org.elasticsearch.action.ActionRequestValidationException +import org.elasticsearch.action.support.master.MasterNodeRequest +import org.elasticsearch.common.io.stream.StreamInput +import org.elasticsearch.common.io.stream.StreamOutput + +// PutComponentTemplateAction +class PutISMTemplateRequest : MasterNodeRequest<PutISMTemplateRequest> { + + val templateName: String + val ismTemplate: ISMTemplate + + constructor(sin: StreamInput) : super(sin) { + templateName = sin.readString() + ismTemplate = ISMTemplate(sin) + } + + override fun writeTo(out: StreamOutput) { + super.writeTo(out) + out.writeString(templateName) + ismTemplate.writeTo(out) + } + + constructor( + templateName: String, + ismTemplate: ISMTemplate + ) : super() { + this.templateName = templateName + this.ismTemplate = ismTemplate + } + + override fun validate(): ActionRequestValidationException? { + return null + } +} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/TransportPutISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/TransportPutISMTemplateAction.kt new file mode 100644 index 000000000..297c1f02a --- /dev/null +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/TransportPutISMTemplateAction.kt @@ -0,0 +1,76 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put + +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ISMTemplateService +import org.apache.logging.log4j.LogManager +import org.elasticsearch.action.ActionListener +import org.elasticsearch.action.support.ActionFilters +import org.elasticsearch.action.support.master.AcknowledgedResponse +import org.elasticsearch.action.support.master.TransportMasterNodeAction +import org.elasticsearch.client.Client +import org.elasticsearch.cluster.ClusterState +import org.elasticsearch.cluster.block.ClusterBlockException +import org.elasticsearch.cluster.block.ClusterBlockLevel +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver +import org.elasticsearch.cluster.service.ClusterService +import org.elasticsearch.common.inject.Inject +import org.elasticsearch.common.io.stream.StreamInput +import org.elasticsearch.common.io.stream.Writeable +import org.elasticsearch.threadpool.ThreadPool +import org.elasticsearch.transport.TransportService + +private val log = LogManager.getLogger(TransportPutISMTemplateAction::class.java) + +class TransportPutISMTemplateAction @Inject constructor( + transportService: TransportService, + clusterService: ClusterService, + threadPool: ThreadPool, + actionFilters: ActionFilters, + indexNameExpressionResolver: IndexNameExpressionResolver, + val client: Client, + val ismTemplateService: ISMTemplateService +) : TransportMasterNodeAction<PutISMTemplateRequest, AcknowledgedResponse>( + PutISMTemplateAction.NAME, + transportService, + clusterService, + threadPool, + actionFilters, + Writeable.Reader { PutISMTemplateRequest(it) }, + indexNameExpressionResolver +) { + /** + * callbacks is inexpensive, this value may be + * {@link org.elasticsearch.threadpool.ThreadPool.Names#SAME SAME} (indicating that the callbacks will run on the same thread + * as the cluster state events are fired with) + */ + override fun executor(): String { + return ThreadPool.Names.SAME + } + + override fun read(sin: StreamInput): AcknowledgedResponse { + return AcknowledgedResponse(sin) + } + + override fun masterOperation(request: PutISMTemplateRequest, state: ClusterState, listener: ActionListener<AcknowledgedResponse>) { + ismTemplateService.putISMTemplate(request.templateName, request.ismTemplate, request.masterNodeTimeout(), listener) + } + + override fun checkBlock(request: PutISMTemplateRequest, state: ClusterState): ClusterBlockException? { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE) + } + +} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt index 32365cb8d..455629aaf 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt @@ -18,7 +18,10 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanageme import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.optionalTimeField import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ChangePolicy +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplateMetadata import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig +import org.elasticsearch.cluster.metadata.Metadata import org.elasticsearch.common.io.stream.StreamInput import org.elasticsearch.common.io.stream.StreamOutput import org.elasticsearch.common.io.stream.Writeable @@ -92,3 +95,21 @@ fun getPartialChangePolicyBuilder( .endObject() .endObject() } + +/** + * return sorted ism templates map saved in cluster metadata + */ +fun Metadata.ismTemplates(): Map<String, ISMTemplate> { + val ismCustomMetadata: ISMTemplateMetadata? = this.custom(ISMTemplateMetadata.TYPE) + return ismCustomMetadata?.ismTemplates?.toSortedMap() ?: emptyMap() +} + +fun Metadata.Builder.putISMTemplate(name: String, template: ISMTemplate, existing: Map<String, ISMTemplate>): Metadata.Builder { + return this.putCustom(ISMTemplateMetadata.TYPE, + ISMTemplateMetadata(existing.plus(name to template))) +} + +fun Metadata.Builder.removeISMTemplate(name: String, existing: Map<String, ISMTemplate>): Metadata.Builder { + return this.putCustom(ISMTemplateMetadata.TYPE, + ISMTemplateMetadata(existing.minus(name))) +} diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt new file mode 100644 index 000000000..7d4cbbad4 --- /dev/null +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt @@ -0,0 +1,5 @@ +package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler + +class ISMTemplateRestAPIIT { + // TODO +} \ No newline at end of file From c93795b3677501b492278d6665b66b1667dd9288 Mon Sep 17 00:00:00 2001 From: bowenlan-amzn <bowenlan@amazon.com> Date: Tue, 8 Dec 2020 16:04:54 -0800 Subject: [PATCH 02/21] start to modify put ism API to observe regulation --- .../ISMTemplateService.kt | 17 ++++- .../indexstatemanagement/model/ISMTemplate.kt | 10 ++- .../model/ISMTemplateMetadata.kt | 2 +- .../delete/DeleteISMTemplateAction.kt | 2 +- .../TransportDeleteISMTemplateAction.kt | 2 +- .../ismtemplate/get/GetISMTemplateRequest.kt | 2 +- .../ismtemplate/get/GetISMTemplateResponse.kt | 6 +- .../get/TransportGetISMTemplateAction.kt | 2 +- .../IndexStateManagementRestTestCase.kt | 73 +++++++++++++++++++ .../indexstatemanagement/TestHelpers.kt | 21 ++++++ .../resthandler/ISMTemplateRestAPIIT.kt | 43 ++++++++++- 11 files changed, 163 insertions(+), 17 deletions(-) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt index 187d222bb..89a4cc376 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt @@ -83,7 +83,9 @@ class ISMTemplateService @Inject constructor( val overlaps = findConflictingISMTemplates(templateName, template.indexPatterns, template.priority, existingTemplates) log.info("find overlapping templates $overlaps") if (overlaps.isNotEmpty()) { - val esg = "new ism template $templateName has index pattern ${template.indexPatterns} matching existing templates ${overlaps.entries.stream().map { "${it.key} => ${it.value}" }.collect(Collectors.joining(","))}, please use a different priority than ${template.priority}" + val esg = "new ism template $templateName has index pattern ${template.indexPatterns} " + + "matching existing templates ${overlaps.entries.stream().map { "${it.key} => ${it.value}" }.collect(Collectors.joining(","))}," + + " please use a different priority than ${template.priority}" throw IllegalArgumentException(esg) } @@ -105,7 +107,8 @@ class ISMTemplateService @Inject constructor( override fun execute(currentState: ClusterState): ClusterState { log.info("service remove template $templateName") val existingTemplates = currentState.metadata.ismTemplates() - return ClusterState.builder(currentState).metadata(Metadata.builder(currentState.metadata).removeISMTemplate(templateName, existingTemplates)).build() + return ClusterState.builder(currentState).metadata(Metadata.builder(currentState.metadata) + .removeISMTemplate(templateName, existingTemplates)).build() } override fun onFailure(source: String, e: Exception) { @@ -129,6 +132,7 @@ class ISMTemplateService @Inject constructor( * filter out older index than template lastUpdateTime */ // findV2Template + @Suppress("ReturnCount") fun findMatchingISMTemplate(ismTemplates: Map<String, ISMTemplate>, indexMetadata: IndexMetadata): String? { val indexName = indexMetadata.index.name @@ -161,6 +165,7 @@ class ISMTemplateService @Inject constructor( return matchedTemplates[winner] } + @Suppress("ComplexMethod") fun validateFormat(templateName: String, indexPatterns: List<String>) { val validationErrors = mutableListOf<String>() if (templateName.contains(" ")) { @@ -216,7 +221,13 @@ class ISMTemplateService @Inject constructor( * @return map of overlapping template name to its index patterns */ // addIndexTemplateV2 findConflictingV2Templates - fun findConflictingISMTemplates(candidate: String, indexPatterns: List<String>, priority: Int, ismTemplates: Map<String, ISMTemplate>): Map<String, List<String>> { + @Suppress("SpreadOperator") + fun findConflictingISMTemplates( + candidate: String, + indexPatterns: List<String>, + priority: Int, + ismTemplates: Map<String, ISMTemplate> + ): Map<String, List<String>> { // focus on template with same priority val ismTemplates = ismTemplates.filter { it.value.priority == priority } val automaton1 = Regex.simpleMatchToAutomaton(*indexPatterns.toTypedArray()) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplate.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplate.kt index 2ea7e32ae..d560080fc 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplate.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplate.kt @@ -15,7 +15,8 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.instant +import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.instant +import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.optionalTimeField import org.apache.logging.log4j.LogManager import org.elasticsearch.cluster.AbstractDiffable import org.elasticsearch.cluster.Diff @@ -51,7 +52,7 @@ data class ISMTemplate( .field(INDEX_PATTERN, indexPatterns) .field(POLICY_ID, policyID) .field(PRIORITY, priority) - .field(LAST_UPDATED_TIME_FIELD, lastUpdatedTime) + .optionalTimeField(LAST_UPDATED_TIME_FIELD, lastUpdatedTime) .endObject() } @@ -77,6 +78,7 @@ data class ISMTemplate( const val PRIORITY = "priority" const val LAST_UPDATED_TIME_FIELD = "last_updated_time" + @Suppress("ComplexMethod") fun parse(xcp: XContentParser): ISMTemplate { val indexPatterns: MutableList<String> = mutableListOf() var policyID: String? = null @@ -84,7 +86,7 @@ data class ISMTemplate( var lastUpdatedTime: Instant? = null log.info("current token ${xcp.currentToken()}") - ensureExpectedToken(Token.START_OBJECT, xcp.nextToken(), xcp::getTokenLocation) + ensureExpectedToken(Token.START_OBJECT, xcp.nextToken(), xcp) while (xcp.nextToken() != Token.END_OBJECT) { val fieldName = xcp.currentName() log.info("parse field name $fieldName") @@ -92,7 +94,7 @@ data class ISMTemplate( when (fieldName) { INDEX_PATTERN -> { - ensureExpectedToken(Token.START_ARRAY, xcp.currentToken(), xcp::getTokenLocation) + ensureExpectedToken(Token.START_ARRAY, xcp.currentToken(), xcp) while (xcp.nextToken() != Token.END_ARRAY) { indexPatterns.add(xcp.text()) log.info("field $indexPatterns") diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateMetadata.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateMetadata.kt index 4082f599f..2197a00ec 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateMetadata.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateMetadata.kt @@ -123,7 +123,7 @@ class ISMTemplateMetadata(val ismTemplates: Map<String, ISMTemplate>): Metadata. fun parse(xcp: XContentParser): ISMTemplateMetadata { val ismTemplates = mutableMapOf<String, ISMTemplate>() log.info("ism template metadata parse, first token ${xcp.currentToken()}") - ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp::getTokenLocation) + ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp) while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { val fieldName = xcp.currentName() log.info("current field name $fieldName") diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateAction.kt index cd0584b7e..6394fd1fd 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateAction.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateAction.kt @@ -23,4 +23,4 @@ class DeleteISMTemplateAction: ActionType<AcknowledgedResponse>(NAME, ::Acknowle val NAME = "cluster:admin/opendistro/ism/templates/remove" val INSTANCE = DeleteISMTemplateAction() } -} \ No newline at end of file +} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/TransportDeleteISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/TransportDeleteISMTemplateAction.kt index c9a0cc64f..bcca9fd34 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/TransportDeleteISMTemplateAction.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/TransportDeleteISMTemplateAction.kt @@ -68,4 +68,4 @@ class TransportDeleteISMTemplateAction @Inject constructor( return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE) } -} \ No newline at end of file +} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequest.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequest.kt index f5c370b6a..ae506db89 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequest.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequest.kt @@ -41,4 +41,4 @@ class GetISMTemplateRequest : MasterNodeRequest<GetISMTemplateRequest> { override fun validate(): ActionRequestValidationException? { return null } -} \ No newline at end of file +} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponse.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponse.kt index be21f7194..3ae0fd302 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponse.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponse.kt @@ -37,14 +37,14 @@ class GetISMTemplateResponse : ActionResponse, ToXContentObject { override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { builder.startObject().startArray(ISM_TEMPLATES) ismTemplates.forEach { (k, v) -> - builder.startObject().field(NAME, k).field(ISM_TEMPLATE, v).endObject() + builder.startObject().field(TEMPLATE_NAME, k).field(ISM_TEMPLATE, v).endObject() } return builder.endArray().endObject() } companion object { val ISM_TEMPLATES = "ism_templates" - val NAME = "name" + val TEMPLATE_NAME = "template_name" val ISM_TEMPLATE = "ism_template" } -} \ No newline at end of file +} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/TransportGetISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/TransportGetISMTemplateAction.kt index 87ef0b9b2..f5cd8929d 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/TransportGetISMTemplateAction.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/TransportGetISMTemplateAction.kt @@ -76,4 +76,4 @@ class TransportGetISMTemplateAction @Inject constructor( return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE) } -} \ No newline at end of file +} diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt index 1577de664..278261045 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt @@ -21,9 +21,11 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlug import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementIndices import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin.Companion.ISM_BASE_URI +import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin.Companion.ISM_TEMPLATE_BASE_URI import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementRestTestCase import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.parseWithType import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ChangePolicy +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.Policy @@ -34,6 +36,9 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagemen import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StateMetaData import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestExplainAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get.GetISMTemplateResponse.Companion.ISM_TEMPLATE +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get.GetISMTemplateResponse.Companion.ISM_TEMPLATES +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get.GetISMTemplateResponse.Companion.TEMPLATE_NAME import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.FAILED_INDICES import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.FAILURES import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.UPDATED_INDICES @@ -696,4 +701,72 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() } return true } + + protected fun createISMTemplate( + name: String, + template: ISMTemplate + ): Response { + val response = createISMTemplateJson(name, template.toJsonString()) + return response + } + + protected fun createISMTemplateJson( + name: String, + templateString: String + ): Response { + val response = client().makeRequest( + "PUT", + "$ISM_TEMPLATE_BASE_URI/$name", + StringEntity(templateString, APPLICATION_JSON) + ) + assertEquals("Unable to create new ISM template", RestStatus.OK, response.restStatus()) + return response + } + + protected fun getISMTemplate(name: String): Map<String, ISMTemplate> { + val response = client().makeRequest("GET", "$ISM_TEMPLATE_BASE_URI/$name") + assertEquals("Unable to get template $name", RestStatus.OK, response.restStatus()) + + val xcp = createParser(XContentType.JSON.xContent(), response.entity.content) + val ismTemplates = mutableMapOf<String, ISMTemplate>() + + ensureExpectedToken(Token.START_OBJECT, xcp.nextToken(), xcp) + + while (xcp.nextToken() != Token.END_OBJECT) { + val fieldName = xcp.currentName() + xcp.nextToken() + + when (fieldName) { + ISM_TEMPLATES -> { + ensureExpectedToken(Token.START_ARRAY, xcp.currentToken(), xcp) + + var templateName: String? = null + var template: ISMTemplate? = null + + while (xcp.nextToken() != Token.END_ARRAY) { + println("t current name: ${xcp.currentName()}") + println("t current token: ${xcp.currentToken()}") + when (xcp.currentName()) { + TEMPLATE_NAME -> { + xcp.nextToken() + templateName = xcp.text() + println("template name $templateName") + } + ISM_TEMPLATE -> { + // xcp.nextToken() + template = ISMTemplate.parse(xcp) + } + } + if (templateName != null && template != null) { + ismTemplates[templateName] = template + templateName = null + template = null + } + } + } + } + } + + return ismTemplates + } } diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/TestHelpers.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/TestHelpers.kt index b081b9d70..fb18170d0 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/TestHelpers.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/TestHelpers.kt @@ -19,6 +19,7 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.string import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ChangePolicy import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.Conditions import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ErrorNotification +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.Policy @@ -59,6 +60,7 @@ import org.elasticsearch.test.rest.ESRestTestCase import java.time.Instant import java.time.ZoneId import java.time.temporal.ChronoUnit +import java.util.regex.Pattern fun randomPolicy( id: String = ESRestTestCase.randomAlphaOfLength(10), @@ -314,6 +316,20 @@ fun randomSweptManagedIndexConfig( ) } +fun randomISMTemplate( + indexPatterns: List<String> = listOf(ESRestTestCase.randomAlphaOfLength(10) + "*"), + policyID: String = ESRestTestCase.randomAlphaOfLength(10), + priority: Int = ESRestTestCase.randomIntBetween(0, 200), + lastUpdatedTime: Instant = Instant.now().truncatedTo(ChronoUnit.MILLIS) +): ISMTemplate { + return ISMTemplate( + indexPatterns = indexPatterns, + policyID = policyID, + priority = priority, + lastUpdatedTime = lastUpdatedTime + ) +} + fun Policy.toJsonString(): String { val builder = XContentFactory.jsonBuilder() return this.toXContent(builder).string() @@ -403,3 +419,8 @@ fun RollupActionConfig.toJsonString(): String { val builder = XContentFactory.jsonBuilder() return this.toXContent(builder, ToXContent.EMPTY_PARAMS).string() } + +fun ISMTemplate.toJsonString(): String { + val builder = XContentFactory.jsonBuilder() + return this.toXContent(builder, ToXContent.EMPTY_PARAMS).string() +} diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt index 7d4cbbad4..7c534183e 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt @@ -1,5 +1,44 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler -class ISMTemplateRestAPIIT { - // TODO +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase +import org.elasticsearch.common.io.Streams +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler +import org.elasticsearch.common.xcontent.NamedXContentRegistry +import org.elasticsearch.common.xcontent.json.JsonXContent +import java.io.InputStreamReader +import java.nio.charset.StandardCharsets + +class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { + fun `test create ISM template`() { + val ismTemplate = """ + { + "index_patterns": ["log*"], + "policy_id": "policy_1", + "priority": 100 + } + """.trimIndent() + val res = createISMTemplateJson("t1", ismTemplate) + // val str1 = res.entity.content.bufferedReader().readText() + // val str2 = Streams.copyToString(InputStreamReader(res.entity.content)) + // val map1 = JsonXContent.jsonXContent + // .createParser(NamedXContentRegistry.EMPTY, + // LoggingDeprecationHandler.INSTANCE, + // res.entity.content).map() + + println("create template response $res") + // println("create template response string1 $str1") + // println("create template response string2 $str2") + // println("create template response map1 $map1") + + val ismTemplatesRes = getISMTemplate("t1") + println("get template parsed response $ismTemplatesRes") + } + + fun `test get ISM template`() { + + } + + fun `test delete ISM template`() { + + } } \ No newline at end of file From 964ef61a6cd095a5245a95bad925fd8b5dd3274f Mon Sep 17 00:00:00 2001 From: bowenlan-amzn <bowenlan@amazon.com> Date: Tue, 8 Dec 2020 16:40:12 -0800 Subject: [PATCH 03/21] conform to regulation add template API --- .../ISMTemplateService.kt | 14 +++- .../indexstatemanagement/model/ISMTemplate.kt | 2 + .../resthandler/RestAddISMTemplateAction.kt | 19 +++++- .../ismtemplate/put/PutISMTemplateAction.kt | 2 +- .../ismtemplate/put/PutISMTemplateResponse.kt | 64 +++++++++++++++++++ .../put/TransportPutISMTemplateAction.kt | 8 +-- 6 files changed, 101 insertions(+), 8 deletions(-) create mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponse.kt diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt index 89a4cc376..80f90a284 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt @@ -17,6 +17,7 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanageme import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put.PutISMTemplateResponse import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.ismTemplates import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.putISMTemplate import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.removeISMTemplate @@ -36,6 +37,7 @@ import org.elasticsearch.common.inject.Inject import org.elasticsearch.common.regex.Regex import org.elasticsearch.common.unit.TimeValue import org.elasticsearch.indices.InvalidIndexTemplateException +import org.elasticsearch.rest.RestStatus import java.util.* import java.util.stream.Collectors @@ -49,7 +51,7 @@ class ISMTemplateService @Inject constructor( * save ISM template to cluster state metadata */ fun putISMTemplate(templateName: String, template: ISMTemplate, masterTimeout: TimeValue, - listener: ActionListener<AcknowledgedResponse>) { + listener: ActionListener<PutISMTemplateResponse>) { clusterService.submitStateUpdateTask( IndexManagementPlugin.PLUGIN_NAME, object : ClusterStateUpdateTask(Priority.NORMAL) { @@ -64,7 +66,15 @@ class ISMTemplateService @Inject constructor( override fun timeout(): TimeValue = masterTimeout override fun clusterStateProcessed(source: String, oldState: ClusterState, newState: ClusterState) { - listener.onResponse(AcknowledgedResponse(true)) + log.info("cluster state processed $source") + var status = RestStatus.CREATED + val oldTemplate = oldState.metadata.ismTemplates()[templateName] + if (oldTemplate != null) { + log.info("old template $oldTemplate") + status = RestStatus.OK + } + // oldTemplate != null ?: { status = RestStatus.OK } + listener.onResponse(PutISMTemplateResponse(templateName, template, status)) } } ) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplate.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplate.kt index d560080fc..47955c24d 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplate.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplate.kt @@ -73,6 +73,8 @@ data class ISMTemplate( } companion object { + const val ISM_TEMPLATE_ID = "template_name" + const val ISM_TEMPLATE_TYPE = "ism_template" const val INDEX_PATTERN = "index_patterns" const val POLICY_ID = "policy_id" const val PRIORITY = "priority" diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestAddISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestAddISMTemplateAction.kt index 18972b254..43f0d1f82 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestAddISMTemplateAction.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestAddISMTemplateAction.kt @@ -15,18 +15,26 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler +import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin.Companion.ISM_TEMPLATE_BASE_URI import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.indexpolicy.IndexPolicyResponse import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put.PutISMTemplateAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put.PutISMTemplateRequest +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put.PutISMTemplateResponse import org.apache.logging.log4j.LogManager import org.elasticsearch.action.support.master.MasterNodeRequest.DEFAULT_MASTER_NODE_TIMEOUT import org.elasticsearch.client.node.NodeClient +import org.elasticsearch.common.xcontent.ToXContent import org.elasticsearch.common.xcontent.XContentHelper import org.elasticsearch.rest.BaseRestHandler +import org.elasticsearch.rest.BytesRestResponse import org.elasticsearch.rest.RestHandler.Route import org.elasticsearch.rest.RestRequest import org.elasticsearch.rest.RestRequest.Method.PUT +import org.elasticsearch.rest.RestResponse +import org.elasticsearch.rest.RestStatus +import org.elasticsearch.rest.action.RestResponseListener import org.elasticsearch.rest.action.RestToXContentListener import java.lang.IllegalArgumentException import java.time.Instant @@ -61,7 +69,16 @@ class RestAddISMTemplateAction : BaseRestHandler() { val addISMTemplateRequest = PutISMTemplateRequest(templateName, ismTemplate).masterNodeTimeout(masterTimeout) return RestChannelConsumer { channel -> - client.execute(PutISMTemplateAction.INSTANCE, addISMTemplateRequest, RestToXContentListener(channel)) + client.execute(PutISMTemplateAction.INSTANCE, addISMTemplateRequest, object : RestResponseListener<PutISMTemplateResponse>(channel) { + override fun buildResponse(response: PutISMTemplateResponse): RestResponse { + val restResponse = BytesRestResponse(response.status, response.toXContent(channel.newBuilder(), ToXContent.EMPTY_PARAMS)) + if (response.status == RestStatus.CREATED) { + val location = "$ISM_TEMPLATE_BASE_URI/${response.id}" + restResponse.addHeader("Location", location) + } + return restResponse + } + }) } } } diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateAction.kt index 25bf14320..b0d6a36d9 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateAction.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateAction.kt @@ -18,7 +18,7 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanageme import org.elasticsearch.action.ActionType import org.elasticsearch.action.support.master.AcknowledgedResponse -class PutISMTemplateAction : ActionType<AcknowledgedResponse>(NAME, ::AcknowledgedResponse) { +class PutISMTemplateAction : ActionType<PutISMTemplateResponse>(NAME, ::PutISMTemplateResponse) { companion object { val NAME = "cluster:admin/opendistro/ism/templates/add" val INSTANCE = PutISMTemplateAction() diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponse.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponse.kt new file mode 100644 index 000000000..fc4a58117 --- /dev/null +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponse.kt @@ -0,0 +1,64 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put + +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate.Companion.ISM_TEMPLATE_ID +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate.Companion.ISM_TEMPLATE_TYPE +import org.elasticsearch.action.ActionResponse +import org.elasticsearch.common.io.stream.StreamInput +import org.elasticsearch.common.io.stream.StreamOutput +import org.elasticsearch.common.xcontent.ToXContent +import org.elasticsearch.common.xcontent.ToXContentObject +import org.elasticsearch.common.xcontent.XContentBuilder +import org.elasticsearch.rest.RestStatus + +class PutISMTemplateResponse : ActionResponse, ToXContentObject { + + val id: String + val template: ISMTemplate + val status: RestStatus + + constructor( + id: String, + template: ISMTemplate, + status: RestStatus + ) : super() { + this.id = id + this.template = template + this.status = status + } + + constructor(sin: StreamInput) : this( + id = sin.readString(), + template = ISMTemplate(sin), + status = sin.readEnum(RestStatus::class.java) + ) + + override fun writeTo(out: StreamOutput) { + out.writeString(id) + template.writeTo(out) + out.writeEnum(status) + } + + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + return builder.startObject() + .field(ISM_TEMPLATE_ID, id) + .field(ISM_TEMPLATE_TYPE, template) + .endObject() + } + +} \ No newline at end of file diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/TransportPutISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/TransportPutISMTemplateAction.kt index 297c1f02a..8155cdd28 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/TransportPutISMTemplateAction.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/TransportPutISMTemplateAction.kt @@ -43,7 +43,7 @@ class TransportPutISMTemplateAction @Inject constructor( indexNameExpressionResolver: IndexNameExpressionResolver, val client: Client, val ismTemplateService: ISMTemplateService -) : TransportMasterNodeAction<PutISMTemplateRequest, AcknowledgedResponse>( +) : TransportMasterNodeAction<PutISMTemplateRequest, PutISMTemplateResponse>( PutISMTemplateAction.NAME, transportService, clusterService, @@ -61,11 +61,11 @@ class TransportPutISMTemplateAction @Inject constructor( return ThreadPool.Names.SAME } - override fun read(sin: StreamInput): AcknowledgedResponse { - return AcknowledgedResponse(sin) + override fun read(sin: StreamInput): PutISMTemplateResponse { + return PutISMTemplateResponse(sin) } - override fun masterOperation(request: PutISMTemplateRequest, state: ClusterState, listener: ActionListener<AcknowledgedResponse>) { + override fun masterOperation(request: PutISMTemplateRequest, state: ClusterState, listener: ActionListener<PutISMTemplateResponse>) { ismTemplateService.putISMTemplate(request.templateName, request.ismTemplate, request.masterNodeTimeout(), listener) } From a29eae1e3d0b09e3bc1616078cdc9b8b5d08a52b Mon Sep 17 00:00:00 2001 From: bowenlan-amzn <bowenlan@amazon.com> Date: Wed, 9 Dec 2020 07:25:58 -0800 Subject: [PATCH 04/21] save progress --- .../IndexStateManagementRestTestCase.kt | 38 ++++++++++++-- .../resthandler/ISMTemplateRestAPIIT.kt | 50 +++++++++++++------ 2 files changed, 70 insertions(+), 18 deletions(-) diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt index 278261045..9a3035ff1 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt @@ -706,8 +706,7 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() name: String, template: ISMTemplate ): Response { - val response = createISMTemplateJson(name, template.toJsonString()) - return response + return createISMTemplateJson(name, template.toJsonString()) } protected fun createISMTemplateJson( @@ -719,11 +718,16 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() "$ISM_TEMPLATE_BASE_URI/$name", StringEntity(templateString, APPLICATION_JSON) ) - assertEquals("Unable to create new ISM template", RestStatus.OK, response.restStatus()) return response } - protected fun getISMTemplate(name: String): Map<String, ISMTemplate> { + protected fun getISMTemplatesMap(name: String): Map<String, Any> { + val response = client().makeRequest("GET", "$ISM_TEMPLATE_BASE_URI/$name") + assertEquals("Unexpected RestStatus", RestStatus.OK, response.restStatus()) + return response.asMap() + } + + protected fun getISMTemplates(name: String): Map<String, ISMTemplate> { val response = client().makeRequest("GET", "$ISM_TEMPLATE_BASE_URI/$name") assertEquals("Unable to get template $name", RestStatus.OK, response.restStatus()) @@ -769,4 +773,30 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() return ismTemplates } + + protected fun assertPredicatesOnISMTemplates( + templatePredicates: List<Pair<String, List<Pair<String, (Any?) -> Boolean>>>>, // name: predicate + response: Map<String, Any?>, + strict: Boolean = true + ) { + val templates = response["ism_templates"] as ArrayList<*> + + templatePredicates.forEach { (name, predicates) -> + // assertTrue("The template: $name was not found in the response: $response", templates.containsKey(name)) + val singleRes = templates[0] as Map<String, Any?> + predicates.forEach { (fieldName, predicate) -> + assertTrue("The key: $fieldName was not found in the response: $singleRes", singleRes.containsKey(fieldName)) + assertTrue("Failed predicate assertion for $fieldName response=($singleRes) predicates=$predicates", predicate(singleRes[fieldName])) + } + } + } + + protected fun assertISMTemplateEquals(expectedISMTemplateMap: ISMTemplate, actualISMTemplateMap: Any?): Boolean { + actualISMTemplateMap as Map<String, Any> + assertEquals(expectedISMTemplateMap.indexPatterns, actualISMTemplateMap[ISMTemplate.INDEX_PATTERN]) + assertEquals(expectedISMTemplateMap.policyID, actualISMTemplateMap[ISMTemplate.POLICY_ID]) + return true + } + + } diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt index 7c534183e..b38a56de5 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt @@ -1,15 +1,16 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase -import org.elasticsearch.common.io.Streams -import org.elasticsearch.common.xcontent.LoggingDeprecationHandler -import org.elasticsearch.common.xcontent.NamedXContentRegistry -import org.elasticsearch.common.xcontent.json.JsonXContent -import java.io.InputStreamReader -import java.nio.charset.StandardCharsets +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate +import com.amazon.opendistroforelasticsearch.indexmanagement.randomInstant +import org.elasticsearch.rest.RestStatus +import java.util.* class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { - fun `test create ISM template`() { + + fun `test ISM template`() { + val templateName = "t1" + println("template name $templateName") val ismTemplate = """ { "index_patterns": ["log*"], @@ -17,7 +18,9 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { "priority": 100 } """.trimIndent() - val res = createISMTemplateJson("t1", ismTemplate) + var res = createISMTemplateJson(templateName, ismTemplate) + assertEquals("Unable to create new ISM template", RestStatus.CREATED, res.restStatus()) + // val str1 = res.entity.content.bufferedReader().readText() // val str2 = Streams.copyToString(InputStreamReader(res.entity.content)) // val map1 = JsonXContent.jsonXContent @@ -30,15 +33,34 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { // println("create template response string2 $str2") // println("create template response map1 $map1") - val ismTemplatesRes = getISMTemplate("t1") - println("get template parsed response $ismTemplatesRes") - } + res = createISMTemplateJson(templateName, ismTemplate) + assertEquals("Unable to update new ISM template", RestStatus.OK, res.restStatus()) - fun `test get ISM template`() { - } + val ismTemplatesRes = getISMTemplates(templateName) + println("get template parsed response $ismTemplatesRes") - fun `test delete ISM template`() { + // createISMTemplateJson("t2", """ + // { + // "index_patterns": ["log*"], + // "policy_id": "policy_1", + // "priority": 50 + // } + // """.trimIndent()) + val mapp1 = getISMTemplatesMap(templateName) + println("get template response map $mapp1") + assertPredicatesOnISMTemplates( + listOf( + templateName to listOf( + "template_name" to templateName::equals, + "ism_template" to fun(template: Any?): Boolean = assertISMTemplateEquals( + ISMTemplate(listOf("log*"), "policy_1", 100, randomInstant()), + template + ) + ) + ), + mapp1 + ) } } \ No newline at end of file From 5eefbe20fc74eae0e0f726a5d34d3d2b5bfc66a4 Mon Sep 17 00:00:00 2001 From: bowenlan-amzn <bowenlan@amazon.com> Date: Wed, 9 Dec 2020 10:17:44 -0800 Subject: [PATCH 05/21] test in progress --- .../ManagedIndexCoordinator.kt | 9 +- .../settings/ManagedIndexSettings.kt | 2 +- .../get/TransportGetISMTemplateAction.kt | 5 +- .../IndexStateManagementRestTestCase.kt | 45 +++-- .../resthandler/ISMTemplateRestAPIIT.kt | 190 ++++++++++++++---- 5 files changed, 185 insertions(+), 66 deletions(-) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt index 54a8f0de0..61f1e653d 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt @@ -284,11 +284,14 @@ class ManagedIndexCoordinator( val updateManagedIndexReqs = mutableListOf<DocWriteRequest<*>>() matchingTemplates.filter { (_, template) -> template != null }.forEach { (index, template) -> - val indexUuids = indexMetadatas[index].indexUUID + val indexUuid = indexMetadatas[index].indexUUID val policyID = templates[template]?.policyID - if (indexUuids != null && policyID != null) { + if (indexUuid != null && policyID != null) { logger.info("create request for index $index matching template $template") - updateManagedIndexReqs.add(managedIndexConfigIndexRequest(index, indexUuids, policyID, jobInterval)) + logger.info("index name is $index") + logger.info("index uuid is $indexUuid") + logger.info("policy id is $policyID") + updateManagedIndexReqs.add(managedIndexConfigIndexRequest(index, indexUuid, policyID, jobInterval)) } } diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/settings/ManagedIndexSettings.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/settings/ManagedIndexSettings.kt index a3c9f8621..a0b67ea6f 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/settings/ManagedIndexSettings.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/settings/ManagedIndexSettings.kt @@ -24,7 +24,7 @@ import java.util.function.Function class ManagedIndexSettings { companion object { const val DEFAULT_ISM_ENABLED = true - const val DEFAULT_JOB_INTERVAL = 5 + const val DEFAULT_JOB_INTERVAL = 1 private val ALLOW_LIST_ALL = ActionConfig.ActionType.values().toList().map { it.type } val ALLOW_LIST_NONE = emptyList<String>() val SNAPSHOT_DENY_LIST_NONE = emptyList<String>() diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/TransportGetISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/TransportGetISMTemplateAction.kt index f5cd8929d..abcb06d69 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/TransportGetISMTemplateAction.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/TransportGetISMTemplateAction.kt @@ -58,8 +58,9 @@ class TransportGetISMTemplateAction @Inject constructor( } val results = mutableMapOf<String, ISMTemplate>() - val templateNames = request.templateNames - templateNames.forEach { name -> + val reqTemplates = request.templateNames + if (allTemplates.isEmpty()) throw ResourceNotFoundException("index template matching ${reqTemplates.toList()} not found") + reqTemplates.forEach { name -> allTemplates.forEach { (templateName, template) -> when { Regex.simpleMatch(name, templateName) -> results[templateName] = template diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt index 9a3035ff1..e576ee46a 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt @@ -74,6 +74,7 @@ import org.elasticsearch.index.seqno.SequenceNumbers import org.elasticsearch.rest.RestRequest import org.elasticsearch.rest.RestStatus import org.elasticsearch.test.ESTestCase +import org.junit.Assert import java.io.IOException import java.time.Duration import java.time.Instant @@ -478,6 +479,8 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() metadata = ManagedIndexMetaData.parse(xcp) } + + println("get back metadata is $metadata") return metadata } @@ -721,14 +724,16 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() return response } - protected fun getISMTemplatesMap(name: String): Map<String, Any> { + protected fun getISMTemplatesAsMap(name: String): Map<String, Any> { val response = client().makeRequest("GET", "$ISM_TEMPLATE_BASE_URI/$name") assertEquals("Unexpected RestStatus", RestStatus.OK, response.restStatus()) return response.asMap() } - protected fun getISMTemplates(name: String): Map<String, ISMTemplate> { - val response = client().makeRequest("GET", "$ISM_TEMPLATE_BASE_URI/$name") + protected fun getISMTemplatesAsObject(name: String?): Map<String, ISMTemplate> { + var endpoint = ISM_TEMPLATE_BASE_URI + if (name != null) endpoint += "/$name" + val response = client().makeRequest("GET", endpoint) assertEquals("Unable to get template $name", RestStatus.OK, response.restStatus()) val xcp = createParser(XContentType.JSON.xContent(), response.entity.content) @@ -774,29 +779,37 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() return ismTemplates } - protected fun assertPredicatesOnISMTemplates( - templatePredicates: List<Pair<String, List<Pair<String, (Any?) -> Boolean>>>>, // name: predicate - response: Map<String, Any?>, - strict: Boolean = true + protected fun assertPredicatesOnISMTemplatesMap( + templatePredicates: List<Pair<String, List<Pair<String, (Any?) -> Boolean>>>>, // response map name: predicate + response: Map<String, Any?> ) { - val templates = response["ism_templates"] as ArrayList<*> + val templates = response["ism_templates"] as ArrayList<Map<String, Any?>> - templatePredicates.forEach { (name, predicates) -> + templatePredicates.forEachIndexed { ind, (_, predicates) -> // assertTrue("The template: $name was not found in the response: $response", templates.containsKey(name)) - val singleRes = templates[0] as Map<String, Any?> + val template = templates[ind] predicates.forEach { (fieldName, predicate) -> - assertTrue("The key: $fieldName was not found in the response: $singleRes", singleRes.containsKey(fieldName)) - assertTrue("Failed predicate assertion for $fieldName response=($singleRes) predicates=$predicates", predicate(singleRes[fieldName])) + assertTrue("The key: $fieldName was not found in the response: $template", template.containsKey(fieldName)) + assertTrue("Failed predicate assertion for $fieldName in response=($template) predicate=$predicate", predicate(template[fieldName])) } } } - protected fun assertISMTemplateEquals(expectedISMTemplateMap: ISMTemplate, actualISMTemplateMap: Any?): Boolean { + protected fun assertISMTemplateEquals(expected: ISMTemplate, actualISMTemplateMap: Any?): Boolean { actualISMTemplateMap as Map<String, Any> - assertEquals(expectedISMTemplateMap.indexPatterns, actualISMTemplateMap[ISMTemplate.INDEX_PATTERN]) - assertEquals(expectedISMTemplateMap.policyID, actualISMTemplateMap[ISMTemplate.POLICY_ID]) + assertEquals(expected.indexPatterns, actualISMTemplateMap[ISMTemplate.INDEX_PATTERN]) + assertEquals(expected.policyID, actualISMTemplateMap[ISMTemplate.POLICY_ID]) + assertEquals(expected.priority, actualISMTemplateMap[ISMTemplate.PRIORITY]) return true } - + protected fun assertISMTemplateEquals(expected: ISMTemplate, actual: ISMTemplate?): Boolean { + assertNotNull(actual) + if (actual != null) { + assertEquals(expected.indexPatterns, actual.indexPatterns) + assertEquals(expected.policyID, actual.policyID) + assertEquals(expected.priority, actual.priority) + } + return true + } } diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt index b38a56de5..c105f5ba5 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt @@ -1,66 +1,168 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler +import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin +import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin.Companion.ISM_TEMPLATE_BASE_URI import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.Policy +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.State +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.action.ReadOnlyActionConfig +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.randomErrorNotification +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings +import com.amazon.opendistroforelasticsearch.indexmanagement.makeRequest import com.amazon.opendistroforelasticsearch.indexmanagement.randomInstant +import com.amazon.opendistroforelasticsearch.indexmanagement.waitFor +import org.elasticsearch.client.ResponseException import org.elasticsearch.rest.RestStatus +import org.elasticsearch.rest.RestRequest.Method.GET +import org.junit.Assert +import java.time.Instant +import java.time.temporal.ChronoUnit import java.util.* class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { + private val testIndexName = javaClass.simpleName.toLowerCase(Locale.ROOT) + fun `test ISM template`() { val templateName = "t1" - println("template name $templateName") - val ismTemplate = """ - { - "index_patterns": ["log*"], - "policy_id": "policy_1", - "priority": 100 - } - """.trimIndent() - var res = createISMTemplateJson(templateName, ismTemplate) + val ismTemp = ISMTemplate(listOf("log*"), "policy_1", 100, randomInstant()) + + var res = createISMTemplate(templateName, ismTemp) assertEquals("Unable to create new ISM template", RestStatus.CREATED, res.restStatus()) - // val str1 = res.entity.content.bufferedReader().readText() - // val str2 = Streams.copyToString(InputStreamReader(res.entity.content)) - // val map1 = JsonXContent.jsonXContent - // .createParser(NamedXContentRegistry.EMPTY, - // LoggingDeprecationHandler.INSTANCE, - // res.entity.content).map() + res = createISMTemplate(templateName, ismTemp) + assertEquals("Unable to update new ISM template", RestStatus.OK, res.restStatus()) - println("create template response $res") - // println("create template response string1 $str1") - // println("create template response string2 $str2") - // println("create template response map1 $map1") + var getRes = getISMTemplatesAsObject(templateName) + assertISMTemplateEquals(ismTemp, getRes[templateName]) - res = createISMTemplateJson(templateName, ismTemplate) - assertEquals("Unable to update new ISM template", RestStatus.OK, res.restStatus()) + val templateName2 = "t2" + val ismTemp2 = ISMTemplate(listOf("trace*"), "policy_1", 100, randomInstant()) + createISMTemplate(templateName2, ismTemp2) + getRes = getISMTemplatesAsObject("$templateName,$templateName2") + val getRes2 = getISMTemplatesAsObject(null) + assertEquals(getRes, getRes2) + assertISMTemplateEquals(ismTemp, getRes[templateName]) + assertISMTemplateEquals(ismTemp2, getRes[templateName2]) + } - val ismTemplatesRes = getISMTemplates(templateName) - println("get template parsed response $ismTemplatesRes") - - // createISMTemplateJson("t2", """ - // { - // "index_patterns": ["log*"], - // "policy_id": "policy_1", - // "priority": 50 - // } - // """.trimIndent()) - - val mapp1 = getISMTemplatesMap(templateName) - println("get template response map $mapp1") - assertPredicatesOnISMTemplates( - listOf( - templateName to listOf( - "template_name" to templateName::equals, - "ism_template" to fun(template: Any?): Boolean = assertISMTemplateEquals( - ISMTemplate(listOf("log*"), "policy_1", 100, randomInstant()), - template + fun `test get not exist template`() { + val tn = "t1" + try { + client().makeRequest(GET.toString(), "${ISM_TEMPLATE_BASE_URI}/$tn") + fail("Expect a failure") + } catch (e: ResponseException) { + assertEquals("Unexpected RestStatus", RestStatus.NOT_FOUND, e.response.restStatus()) + val actualMessage = e.response.asMap() + val expectErrorMessage = mapOf( + "error" to mapOf( + "root_cause" to listOf<Map<String, Any>>( + mapOf("type" to "resource_not_found_exception", "reason" to "index template matching [$tn] not found") + ), + "type" to "resource_not_found_exception", + "reason" to "index template matching [$tn] not found" + ), + "status" to 404 + ) + assertEquals(expectErrorMessage, actualMessage) + } + } + + fun `test add template with invalid index pattern`() { + val tn = "t1" + try { + val ismTemp = ISMTemplate(listOf(" "), "policy_1", 100, randomInstant()) + createISMTemplate(tn, ismTemp) + fail("Expect a failure") + } catch (e: ResponseException) { + assertEquals("Unexpected RestStatus", RestStatus.BAD_REQUEST, e.response.restStatus()) + val actualMessage = e.response.asMap() + val expectErrorMessage = mapOf( + "error" to mapOf( + "reason" to "index_template [$tn] invalid, cause [Validation Failed: 1: index_patterns [ ] must not contain a space;2: index_pattern [ ] must not contain the following characters [ , \", *, \\, <, |, ,, >, /, ?];]", + "type" to "invalid_index_template_exception", + "root_cause" to listOf<Map<String, Any>>( + mapOf("reason" to "index_template [$tn] invalid, cause [Validation Failed: 1: index_patterns [ ] must not contain a space;2: index_pattern [ ] must not contain the following characters [ , \", *, \\, <, |, ,, >, /, ?];]", + "type" to "invalid_index_template_exception") ) - ) - ), - mapp1 + ), + "status" to 400 + ) + assertEquals(expectErrorMessage, actualMessage) + } + } + + fun `test add template with overlapping index pattern`() { + val tn = "t2" + try { + val ismTemp = ISMTemplate(listOf("log*"), "policy_1", 100, randomInstant()) + createISMTemplate("t1", ismTemp) + createISMTemplate(tn, ismTemp) + fail("Expect a failure") + } catch (e: ResponseException) { + assertEquals("Unexpected RestStatus", RestStatus.BAD_REQUEST, e.response.restStatus()) + val actualMessage = e.response.asMap() + val expectErrorMessage = mapOf( + "error" to mapOf( + "reason" to "new ism template $tn has index pattern [log*] matching existing templates t1 => [log*], please use a different priority than 100", + "type" to "illegal_argument_exception", + "root_cause" to listOf<Map<String, Any>>( + mapOf("reason" to "new ism template $tn has index pattern [log*] matching existing templates t1 => [log*], please use a different priority than 100", + "type" to "illegal_argument_exception") + ) + ), + "status" to 400 + ) + assertEquals(expectErrorMessage, actualMessage) + } + } + + fun `test ism template managing index`() { + val indexName1 = "log-000001" + val indexName2 = "log-000002" + val policyID = "${testIndexName}_testPolicyName_1" + + createIndex(indexName1) + val templateName = "t1" + val ismTemp = ISMTemplate(listOf("log*"), policyID, 100, randomInstant()) + createISMTemplate(templateName, ismTemp) + + println("ism template: ${getISMTemplatesAsObject(null)}") + + val actionConfig = ReadOnlyActionConfig(0) + val states = listOf( + State("ReadOnlyState", listOf(actionConfig), listOf()) ) + val policy = Policy( + id = policyID, + description = "$testIndexName description", + schemaVersion = 1L, + lastUpdatedTime = Instant.now().truncatedTo(ChronoUnit.MILLIS), + errorNotification = randomErrorNotification(), + defaultState = states[0].name, + states = states + ) + createPolicy(policy, policyID) + createIndex(indexName2) + + val managedIndexConfig = getExistingManagedIndexConfig(indexName2) + println("job doc is $managedIndexConfig") + // updateManagedIndexConfigStartTime(managedIndexConfig) + // + // // TODO problem for policyID value + // waitFor { assertEquals(policyID, getExplainManagedIndexMetaData(indexName2).policyID) } + // + // // only index create after template can be managed + // assertPredicatesOnMetaData( + // listOf(indexName1 to listOf(ManagedIndexSettings.POLICY_ID.key to fun(policyID: Any?): Boolean = policyID == null)), + // getExplainMap(indexName1), + // true + // ) + + // hidden index will not be manage + } + } \ No newline at end of file From 2b1ac28dde620f1ca46d8d2bb113b9edccc567b8 Mon Sep 17 00:00:00 2001 From: bowenlan-amzn <bowenlan@amazon.com> Date: Wed, 9 Dec 2020 16:13:52 -0800 Subject: [PATCH 06/21] draft IT --- .../ISMTemplateService.kt | 4 +- .../ManagedIndexCoordinator.kt | 2 + .../resthandler/ISMTemplateRestAPIIT.kt | 38 +++++++++++-------- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt index 80f90a284..f6f6a89e5 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt @@ -152,9 +152,7 @@ class ISMTemplateService @Inject constructor( if (isHidden) return null val ismTemplates = ismTemplates.filter { (_, template) -> - log.info("template last update time: ${template.lastUpdatedTime.toEpochMilli()}") - log.info("index create time: ${indexMetadata.creationDate}") - log.info("is template older? ${template.lastUpdatedTime.toEpochMilli() < indexMetadata.creationDate}") + log.info("index create after template? ${template.lastUpdatedTime.toEpochMilli() < indexMetadata.creationDate}") template.lastUpdatedTime.toEpochMilli() < indexMetadata.creationDate } diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt index 61f1e653d..eac5db355 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt @@ -230,6 +230,7 @@ class ManagedIndexCoordinator( @OpenForTesting suspend fun sweepClusterChangedEvent(event: ClusterChangedEvent) { + logger.info("start sweep cluster changed event") val indicesDeletedRequests = event.indicesDeleted() .filter { event.previousState().metadata().index(it)?.getPolicyID() != null } .map { deleteManagedIndexRequest(it.uuid) } @@ -294,6 +295,7 @@ class ManagedIndexCoordinator( updateManagedIndexReqs.add(managedIndexConfigIndexRequest(index, indexUuid, policyID, jobInterval)) } } + logger.info("size of matching template req ${updateManagedIndexReqs.size}") return updateManagedIndexReqs } diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt index c105f5ba5..d320fdb79 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt @@ -13,6 +13,7 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.makeRequest import com.amazon.opendistroforelasticsearch.indexmanagement.randomInstant import com.amazon.opendistroforelasticsearch.indexmanagement.waitFor import org.elasticsearch.client.ResponseException +import org.elasticsearch.common.settings.Settings import org.elasticsearch.rest.RestStatus import org.elasticsearch.rest.RestRequest.Method.GET import org.junit.Assert @@ -122,9 +123,11 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { fun `test ism template managing index`() { val indexName1 = "log-000001" val indexName2 = "log-000002" + val indexName3 = "log-000003" val policyID = "${testIndexName}_testPolicyName_1" - createIndex(indexName1) + // need to specify policyID null, can remove after policyID deprecated + createIndex(indexName1, null) val templateName = "t1" val ismTemp = ISMTemplate(listOf("log*"), policyID, 100, randomInstant()) createISMTemplate(templateName, ismTemp) @@ -145,24 +148,29 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { states = states ) createPolicy(policy, policyID) - createIndex(indexName2) + createIndex(indexName2, null) + createIndex(indexName3, Settings.builder().put("index.hidden", true).build()) val managedIndexConfig = getExistingManagedIndexConfig(indexName2) - println("job doc is $managedIndexConfig") - // updateManagedIndexConfigStartTime(managedIndexConfig) - // - // // TODO problem for policyID value - // waitFor { assertEquals(policyID, getExplainManagedIndexMetaData(indexName2).policyID) } - // - // // only index create after template can be managed - // assertPredicatesOnMetaData( - // listOf(indexName1 to listOf(ManagedIndexSettings.POLICY_ID.key to fun(policyID: Any?): Boolean = policyID == null)), - // getExplainMap(indexName1), - // true - // ) + updateManagedIndexConfigStartTime(managedIndexConfig) + waitFor { assertEquals(policyID, getExplainManagedIndexMetaData(indexName2).policyID) } + + // only index create after template can be managed + assertPredicatesOnMetaData( + listOf(indexName1 to listOf(ManagedIndexSettings.POLICY_ID.key to fun(policyID: Any?): Boolean = policyID == null)), + getExplainMap(indexName1), + true + ) + assertNull(getManagedIndexConfig(indexName1)) - // hidden index will not be manage + // hidden index will not be manage + assertPredicatesOnMetaData( + listOf(indexName1 to listOf(ManagedIndexSettings.POLICY_ID.key to fun(policyID: Any?): Boolean = policyID == null)), + getExplainMap(indexName1), + true + ) + assertNull(getManagedIndexConfig(indexName3)) } } \ No newline at end of file From 31889dac9dc58dc43b6bcc115730feb4566b62b5 Mon Sep 17 00:00:00 2001 From: bowenlan-amzn <bowenlan@amazon.com> Date: Wed, 9 Dec 2020 16:38:08 -0800 Subject: [PATCH 07/21] simple tests for request response --- .../transport/action/ActionTests.kt | 18 ++++++++ .../delete/DeleteISMTemplateRequestTests.kt | 34 +++++++++++++++ .../get/GetISMTemplateRequestTests.kt | 34 +++++++++++++++ .../get/GetISMTemplateResponseTests.kt | 36 ++++++++++++++++ .../put/PutISMTemplateRequestTests.kt | 38 +++++++++++++++++ .../put/PutISMTemplateResponseTests.kt | 41 +++++++++++++++++++ 6 files changed, 201 insertions(+) create mode 100644 src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequestTests.kt create mode 100644 src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequestTests.kt create mode 100644 src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponseTests.kt create mode 100644 src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateRequestTests.kt create mode 100644 src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponseTests.kt diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ActionTests.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ActionTests.kt index 003329b8f..f5a9459e9 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ActionTests.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ActionTests.kt @@ -21,6 +21,9 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagemen import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.indexpolicy.IndexPolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.explain.ExplainAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.getpolicy.GetPolicyAction +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.delete.DeleteISMTemplateAction +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get.GetISMTemplateAction +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put.PutISMTemplateAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.removepolicy.RemovePolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.retryfailedmanagedindex.RetryFailedManagedIndexAction import org.elasticsearch.test.ESTestCase @@ -65,4 +68,19 @@ class ActionTests : ESTestCase() { assertNotNull(GetPolicyAction.NAME) assertEquals(GetPolicyAction.INSTANCE.name(), GetPolicyAction.NAME) } + + fun `test get template action name`() { + assertNotNull(GetISMTemplateAction.NAME) + assertEquals(GetISMTemplateAction.INSTANCE.name(), GetISMTemplateAction.NAME) + } + + fun `test put template action name`() { + assertNotNull(PutISMTemplateAction.NAME) + assertEquals(PutISMTemplateAction.INSTANCE.name(), PutISMTemplateAction.NAME) + } + + fun `test delete template action name`() { + assertNotNull(DeleteISMTemplateAction.NAME) + assertEquals(DeleteISMTemplateAction.INSTANCE.name(), DeleteISMTemplateAction.NAME) + } } diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequestTests.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequestTests.kt new file mode 100644 index 000000000..78759c002 --- /dev/null +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequestTests.kt @@ -0,0 +1,34 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.delete + +import org.elasticsearch.common.io.stream.BytesStreamOutput +import org.elasticsearch.common.io.stream.StreamInput +import org.elasticsearch.test.ESTestCase + +class DeleteISMTemplateRequestTests : ESTestCase() { + + fun `test delete template request`() { + val templateName = "t1" + val req = DeleteISMTemplateRequest(templateName) + + val out = BytesStreamOutput() + req.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newReq = DeleteISMTemplateRequest(sin) + assertEquals(templateName, newReq.templateName) + } +} \ No newline at end of file diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequestTests.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequestTests.kt new file mode 100644 index 000000000..5e46e3732 --- /dev/null +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequestTests.kt @@ -0,0 +1,34 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get + +import org.elasticsearch.common.io.stream.BytesStreamOutput +import org.elasticsearch.common.io.stream.StreamInput +import org.elasticsearch.test.ESTestCase + +class GetISMTemplateRequestTests : ESTestCase() { + + fun `test get templates request`() { + val templateNames = arrayOf("t1") + val req = GetISMTemplateRequest(templateNames) + + val out = BytesStreamOutput() + req.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newReq = GetISMTemplateRequest(sin) + assertEquals(templateNames, newReq.templateNames) + } +} \ No newline at end of file diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponseTests.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponseTests.kt new file mode 100644 index 000000000..a9ca3c2ad --- /dev/null +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponseTests.kt @@ -0,0 +1,36 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get + +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate +import com.amazon.opendistroforelasticsearch.indexmanagement.randomInstant +import org.elasticsearch.common.io.stream.BytesStreamOutput +import org.elasticsearch.common.io.stream.StreamInput +import org.elasticsearch.test.ESTestCase + +class GetISMTemplateResponseTests : ESTestCase() { + + fun `test get templates response`() { + val ismTemplates = mapOf("t1" to ISMTemplate(listOf("log*"), "policy_1", 100, randomInstant())) + val req = GetISMTemplateResponse(ismTemplates) + + val out = BytesStreamOutput() + req.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newReq = GetISMTemplateResponse(sin) + assertEquals(ismTemplates, newReq.ismTemplates) + } +} \ No newline at end of file diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateRequestTests.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateRequestTests.kt new file mode 100644 index 000000000..b8747eb4e --- /dev/null +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateRequestTests.kt @@ -0,0 +1,38 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put + +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate +import com.amazon.opendistroforelasticsearch.indexmanagement.randomInstant +import org.elasticsearch.common.io.stream.BytesStreamOutput +import org.elasticsearch.common.io.stream.StreamInput +import org.elasticsearch.test.ESTestCase + +class PutISMTemplateRequestTests : ESTestCase() { + + fun `test put template request`() { + val templateName = "t1" + val ismTemplate = ISMTemplate(listOf("log*"), "policy_1", 100, randomInstant()) + val req = PutISMTemplateRequest(templateName, ismTemplate) + + val out = BytesStreamOutput() + req.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newReq = PutISMTemplateRequest(sin) + assertEquals(templateName, newReq.templateName) + assertEquals(ismTemplate, newReq.ismTemplate) + } +} \ No newline at end of file diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponseTests.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponseTests.kt new file mode 100644 index 000000000..031923c5a --- /dev/null +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponseTests.kt @@ -0,0 +1,41 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put + +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate +import com.amazon.opendistroforelasticsearch.indexmanagement.randomInstant +import org.elasticsearch.common.io.stream.BytesStreamOutput +import org.elasticsearch.common.io.stream.StreamInput +import org.elasticsearch.rest.RestStatus +import org.elasticsearch.test.ESTestCase + +class PutISMTemplateResponseTests : ESTestCase() { + + fun `test put template response`() { + val id = "t1" + val template = ISMTemplate(listOf("log*"), "policy_1", 100, randomInstant()) + val status = RestStatus.OK + val req = PutISMTemplateResponse(id, template, status) + + val out = BytesStreamOutput() + req.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newReq = PutISMTemplateResponse(sin) + assertEquals(id, newReq.id) + assertEquals(template, newReq.template) + assertEquals(status, newReq.status) + } +} \ No newline at end of file From 80d4519e9191bf163d45c8a32bdf73c0b0fdcae0 Mon Sep 17 00:00:00 2001 From: bowenlan-amzn <bowenlan@amazon.com> Date: Thu, 10 Dec 2020 08:54:59 -0800 Subject: [PATCH 08/21] ktlint --- .../indexmanagement/IndexManagementPlugin.kt | 5 ++-- .../ISMTemplateService.kt | 12 +++++--- .../model/ISMTemplateMetadata.kt | 11 ++----- .../resthandler/RestAddISMTemplateAction.kt | 3 -- .../RestDeleteISMTemplateAction.kt | 3 +- .../resthandler/RestGetISMTemplateAction.kt | 1 - .../delete/DeleteISMTemplateAction.kt | 2 +- .../TransportDeleteISMTemplateAction.kt | 29 +++++++++--------- .../ismtemplate/get/GetISMTemplateResponse.kt | 1 - .../get/TransportGetISMTemplateAction.kt | 30 +++++++++---------- .../ismtemplate/put/PutISMTemplateAction.kt | 1 - .../ismtemplate/put/PutISMTemplateResponse.kt | 3 +- .../put/TransportPutISMTemplateAction.kt | 30 +++++++++---------- .../IndexStateManagementRestTestCase.kt | 1 - .../indexstatemanagement/TestHelpers.kt | 3 +- .../resthandler/ISMTemplateRestAPIIT.kt | 11 ++----- 16 files changed, 62 insertions(+), 84 deletions(-) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/IndexManagementPlugin.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/IndexManagementPlugin.kt index 859799346..1bf40a949 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/IndexManagementPlugin.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/IndexManagementPlugin.kt @@ -112,7 +112,6 @@ import org.elasticsearch.common.settings.IndexScopedSettings import org.elasticsearch.common.settings.Setting import org.elasticsearch.common.settings.Settings import org.elasticsearch.common.settings.SettingsFilter -import org.elasticsearch.common.xcontent.ContextParser import org.elasticsearch.common.util.concurrent.ThreadContext import org.elasticsearch.common.xcontent.NamedXContentRegistry import org.elasticsearch.common.xcontent.XContentParser.Token @@ -340,12 +339,12 @@ internal class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, Act val ismTemplateEntry = NamedWriteableRegistry.Entry( Metadata.Custom::class.java, ISMTemplateMetadata.TYPE, - Writeable.Reader{ sin -> ISMTemplateMetadata(sin) } + Writeable.Reader { sin -> ISMTemplateMetadata(sin) } ) val ismTemplateEntry2 = NamedWriteableRegistry.Entry( NamedDiff::class.java, ISMTemplateMetadata.TYPE, - Writeable.Reader{ sin -> ISMTemplateMetadata.readDiffFrom(sin) } + Writeable.Reader { sin -> ISMTemplateMetadata.readDiffFrom(sin) } ) entries.add(ismTemplateEntry) entries.add(ismTemplateEntry2) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt index f6f6a89e5..a8188cef7 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt @@ -45,13 +45,17 @@ private val log = LogManager.getLogger(ISMTemplateService::class.java) // MetadataIndexTemplateService class ISMTemplateService @Inject constructor( - val clusterService: ClusterService + val clusterService: ClusterService ) { /** * save ISM template to cluster state metadata */ - fun putISMTemplate(templateName: String, template: ISMTemplate, masterTimeout: TimeValue, - listener: ActionListener<PutISMTemplateResponse>) { + fun putISMTemplate( + templateName: String, + template: ISMTemplate, + masterTimeout: TimeValue, + listener: ActionListener<PutISMTemplateResponse> + ) { clusterService.submitStateUpdateTask( IndexManagementPlugin.PLUGIN_NAME, object : ClusterStateUpdateTask(Priority.NORMAL) { @@ -248,7 +252,7 @@ class ISMTemplateService @Inject constructor( } } overlappingTemplates.remove(candidate) - return overlappingTemplates + return overlappingTemplates } } } diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateMetadata.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateMetadata.kt index 2197a00ec..308724c43 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateMetadata.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateMetadata.kt @@ -17,11 +17,9 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanageme import org.apache.logging.log4j.LogManager import org.elasticsearch.Version -import org.elasticsearch.cluster.AbstractNamedDiffable import org.elasticsearch.cluster.Diff import org.elasticsearch.cluster.DiffableUtils import org.elasticsearch.cluster.NamedDiff -import org.elasticsearch.cluster.metadata.ComponentTemplate import org.elasticsearch.cluster.metadata.Metadata import org.elasticsearch.common.ParseField import org.elasticsearch.common.io.stream.StreamInput @@ -32,10 +30,8 @@ import org.elasticsearch.common.xcontent.ContextParser import org.elasticsearch.common.xcontent.ToXContent import org.elasticsearch.common.xcontent.XContentBuilder import org.elasticsearch.common.xcontent.XContentParser -import org.elasticsearch.common.xcontent.XContentParserUtils import org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken -import java.util.* -import java.util.stream.Stream +import java.util.EnumSet private val log = LogManager.getLogger(ISMTemplateMetadata::class.java) @@ -44,7 +40,7 @@ private val log = LogManager.getLogger(ISMTemplateMetadata::class.java) */ // ComponentTemplateMetadata // EnrichMetadata -class ISMTemplateMetadata(val ismTemplates: Map<String, ISMTemplate>): Metadata.Custom { +class ISMTemplateMetadata(val ismTemplates: Map<String, ISMTemplate>) : Metadata.Custom { constructor(sin: StreamInput) : this( sin.readMap(StreamInput::readString, ::ISMTemplate) @@ -74,7 +70,7 @@ class ISMTemplateMetadata(val ismTemplates: Map<String, ISMTemplate>): Metadata. override fun context(): EnumSet<Metadata.XContentContext> = Metadata.ALL_CONTEXTS - class ISMTemplateMetadataDiff: NamedDiff<Metadata.Custom> { + class ISMTemplateMetadataDiff : NamedDiff<Metadata.Custom> { val ismTemplateDiff: Diff<Map<String, ISMTemplate>> @@ -96,7 +92,6 @@ class ISMTemplateMetadata(val ismTemplates: Map<String, ISMTemplate>): Metadata. override fun apply(part: Metadata.Custom): Metadata.Custom { return ISMTemplateMetadata(ismTemplateDiff.apply((part as ISMTemplateMetadata).ismTemplates)) } - } companion object { diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestAddISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestAddISMTemplateAction.kt index 43f0d1f82..2582c14c2 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestAddISMTemplateAction.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestAddISMTemplateAction.kt @@ -15,10 +15,8 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler -import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin.Companion.ISM_TEMPLATE_BASE_URI import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.indexpolicy.IndexPolicyResponse import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put.PutISMTemplateAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put.PutISMTemplateRequest import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put.PutISMTemplateResponse @@ -35,7 +33,6 @@ import org.elasticsearch.rest.RestRequest.Method.PUT import org.elasticsearch.rest.RestResponse import org.elasticsearch.rest.RestStatus import org.elasticsearch.rest.action.RestResponseListener -import org.elasticsearch.rest.action.RestToXContentListener import java.lang.IllegalArgumentException import java.time.Instant diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestDeleteISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestDeleteISMTemplateAction.kt index 6c7b84c17..74c709fe8 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestDeleteISMTemplateAction.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestDeleteISMTemplateAction.kt @@ -21,7 +21,6 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagemen import org.apache.logging.log4j.LogManager import org.elasticsearch.client.node.NodeClient import org.elasticsearch.rest.BaseRestHandler -import org.elasticsearch.rest.RestHandler import org.elasticsearch.rest.RestHandler.Route import org.elasticsearch.rest.RestRequest import org.elasticsearch.rest.RestRequest.Method.DELETE @@ -33,7 +32,7 @@ private val log = LogManager.getLogger(RestDeleteISMTemplateAction::class.java) class RestDeleteISMTemplateAction : BaseRestHandler() { override fun routes(): List<Route> { return listOf( - Route(DELETE, "${ISM_TEMPLATE_BASE_URI}/{templateID}") + Route(DELETE, "$ISM_TEMPLATE_BASE_URI/{templateID}") ) } diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestGetISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestGetISMTemplateAction.kt index 53ad40610..0c8c2a05c 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestGetISMTemplateAction.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestGetISMTemplateAction.kt @@ -23,7 +23,6 @@ import org.elasticsearch.action.support.master.MasterNodeRequest.DEFAULT_MASTER_ import org.elasticsearch.client.node.NodeClient import org.elasticsearch.common.Strings import org.elasticsearch.rest.BaseRestHandler -import org.elasticsearch.rest.RestHandler import org.elasticsearch.rest.RestHandler.Route import org.elasticsearch.rest.RestRequest import org.elasticsearch.rest.RestRequest.Method.GET diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateAction.kt index 6394fd1fd..b90840132 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateAction.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateAction.kt @@ -18,7 +18,7 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanageme import org.elasticsearch.action.ActionType import org.elasticsearch.action.support.master.AcknowledgedResponse -class DeleteISMTemplateAction: ActionType<AcknowledgedResponse>(NAME, ::AcknowledgedResponse) { +class DeleteISMTemplateAction : ActionType<AcknowledgedResponse>(NAME, ::AcknowledgedResponse) { companion object { val NAME = "cluster:admin/opendistro/ism/templates/remove" val INSTANCE = DeleteISMTemplateAction() diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/TransportDeleteISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/TransportDeleteISMTemplateAction.kt index bcca9fd34..ad79ebb13 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/TransportDeleteISMTemplateAction.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/TransportDeleteISMTemplateAction.kt @@ -36,21 +36,21 @@ import org.elasticsearch.transport.TransportService private val log = LogManager.getLogger(TransportDeleteISMTemplateAction::class.java) class TransportDeleteISMTemplateAction @Inject constructor( - transportService: TransportService, - clusterService: ClusterService, - threadPool: ThreadPool, - actionFilters: ActionFilters, - indexNameExpressionResolver: IndexNameExpressionResolver, - val client: Client, - val ismTemplateService: ISMTemplateService + transportService: TransportService, + clusterService: ClusterService, + threadPool: ThreadPool, + actionFilters: ActionFilters, + indexNameExpressionResolver: IndexNameExpressionResolver, + val client: Client, + val ismTemplateService: ISMTemplateService ) : TransportMasterNodeAction<DeleteISMTemplateRequest, AcknowledgedResponse>( - DeleteISMTemplateAction.NAME, - transportService, - clusterService, - threadPool, - actionFilters, - Writeable.Reader { DeleteISMTemplateRequest(it) }, - indexNameExpressionResolver + DeleteISMTemplateAction.NAME, + transportService, + clusterService, + threadPool, + actionFilters, + Writeable.Reader { DeleteISMTemplateRequest(it) }, + indexNameExpressionResolver ) { override fun executor(): String { return ThreadPool.Names.SAME @@ -67,5 +67,4 @@ class TransportDeleteISMTemplateAction @Inject constructor( override fun checkBlock(request: DeleteISMTemplateRequest, state: ClusterState): ClusterBlockException? { return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE) } - } diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponse.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponse.kt index 3ae0fd302..e28249746 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponse.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponse.kt @@ -2,7 +2,6 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanageme import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate import org.elasticsearch.action.ActionResponse -import org.elasticsearch.common.ParseField import org.elasticsearch.common.io.stream.StreamInput import org.elasticsearch.common.io.stream.StreamOutput import org.elasticsearch.common.xcontent.ToXContent diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/TransportGetISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/TransportGetISMTemplateAction.kt index abcb06d69..f28d9a472 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/TransportGetISMTemplateAction.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/TransportGetISMTemplateAction.kt @@ -21,26 +21,25 @@ import org.elasticsearch.common.regex.Regex import org.elasticsearch.threadpool.ThreadPool import org.elasticsearch.transport.TransportService - private val log = LogManager.getLogger(TransportGetISMTemplateAction::class.java) // TransportGetComposableIndexTemplateAction class TransportGetISMTemplateAction @Inject constructor( - transportService: TransportService, - clusterService: ClusterService, - threadPool: ThreadPool, - actionFilters: ActionFilters, - indexNameExpressionResolver: IndexNameExpressionResolver, - val client: Client, - val ismTemplateService: ISMTemplateService + transportService: TransportService, + clusterService: ClusterService, + threadPool: ThreadPool, + actionFilters: ActionFilters, + indexNameExpressionResolver: IndexNameExpressionResolver, + val client: Client, + val ismTemplateService: ISMTemplateService ) : TransportMasterNodeAction<GetISMTemplateRequest, GetISMTemplateResponse>( - GetISMTemplateAction.NAME, - transportService, - clusterService, - threadPool, - actionFilters, - Writeable.Reader { GetISMTemplateRequest(it) }, - indexNameExpressionResolver + GetISMTemplateAction.NAME, + transportService, + clusterService, + threadPool, + actionFilters, + Writeable.Reader { GetISMTemplateRequest(it) }, + indexNameExpressionResolver ) { override fun executor(): String { return ThreadPool.Names.SAME @@ -76,5 +75,4 @@ class TransportGetISMTemplateAction @Inject constructor( override fun checkBlock(request: GetISMTemplateRequest, state: ClusterState): ClusterBlockException? { return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE) } - } diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateAction.kt index b0d6a36d9..af65d2c14 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateAction.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateAction.kt @@ -16,7 +16,6 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put import org.elasticsearch.action.ActionType -import org.elasticsearch.action.support.master.AcknowledgedResponse class PutISMTemplateAction : ActionType<PutISMTemplateResponse>(NAME, ::PutISMTemplateResponse) { companion object { diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponse.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponse.kt index fc4a58117..e63915c03 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponse.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponse.kt @@ -60,5 +60,4 @@ class PutISMTemplateResponse : ActionResponse, ToXContentObject { .field(ISM_TEMPLATE_TYPE, template) .endObject() } - -} \ No newline at end of file +} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/TransportPutISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/TransportPutISMTemplateAction.kt index 8155cdd28..bfac1e6bc 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/TransportPutISMTemplateAction.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/TransportPutISMTemplateAction.kt @@ -19,7 +19,6 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagemen import org.apache.logging.log4j.LogManager import org.elasticsearch.action.ActionListener import org.elasticsearch.action.support.ActionFilters -import org.elasticsearch.action.support.master.AcknowledgedResponse import org.elasticsearch.action.support.master.TransportMasterNodeAction import org.elasticsearch.client.Client import org.elasticsearch.cluster.ClusterState @@ -36,21 +35,21 @@ import org.elasticsearch.transport.TransportService private val log = LogManager.getLogger(TransportPutISMTemplateAction::class.java) class TransportPutISMTemplateAction @Inject constructor( - transportService: TransportService, - clusterService: ClusterService, - threadPool: ThreadPool, - actionFilters: ActionFilters, - indexNameExpressionResolver: IndexNameExpressionResolver, - val client: Client, - val ismTemplateService: ISMTemplateService + transportService: TransportService, + clusterService: ClusterService, + threadPool: ThreadPool, + actionFilters: ActionFilters, + indexNameExpressionResolver: IndexNameExpressionResolver, + val client: Client, + val ismTemplateService: ISMTemplateService ) : TransportMasterNodeAction<PutISMTemplateRequest, PutISMTemplateResponse>( - PutISMTemplateAction.NAME, - transportService, - clusterService, - threadPool, - actionFilters, - Writeable.Reader { PutISMTemplateRequest(it) }, - indexNameExpressionResolver + PutISMTemplateAction.NAME, + transportService, + clusterService, + threadPool, + actionFilters, + Writeable.Reader { PutISMTemplateRequest(it) }, + indexNameExpressionResolver ) { /** * callbacks is inexpensive, this value may be @@ -72,5 +71,4 @@ class TransportPutISMTemplateAction @Inject constructor( override fun checkBlock(request: PutISMTemplateRequest, state: ClusterState): ClusterBlockException? { return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE) } - } diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt index e576ee46a..2b4e3f4cf 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt @@ -74,7 +74,6 @@ import org.elasticsearch.index.seqno.SequenceNumbers import org.elasticsearch.rest.RestRequest import org.elasticsearch.rest.RestStatus import org.elasticsearch.test.ESTestCase -import org.junit.Assert import java.io.IOException import java.time.Duration import java.time.Instant diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/TestHelpers.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/TestHelpers.kt index fb18170d0..37e3557c6 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/TestHelpers.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/TestHelpers.kt @@ -60,7 +60,6 @@ import org.elasticsearch.test.rest.ESRestTestCase import java.time.Instant import java.time.ZoneId import java.time.temporal.ChronoUnit -import java.util.regex.Pattern fun randomPolicy( id: String = ESRestTestCase.randomAlphaOfLength(10), @@ -422,5 +421,5 @@ fun RollupActionConfig.toJsonString(): String { fun ISMTemplate.toJsonString(): String { val builder = XContentFactory.jsonBuilder() - return this.toXContent(builder, ToXContent.EMPTY_PARAMS).string() + return this.toXContent(builder, ToXContent.EMPTY_PARAMS).string() } diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt index d320fdb79..e4eb72251 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt @@ -1,6 +1,5 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler -import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin.Companion.ISM_TEMPLATE_BASE_URI import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate @@ -16,10 +15,9 @@ import org.elasticsearch.client.ResponseException import org.elasticsearch.common.settings.Settings import org.elasticsearch.rest.RestStatus import org.elasticsearch.rest.RestRequest.Method.GET -import org.junit.Assert import java.time.Instant import java.time.temporal.ChronoUnit -import java.util.* +import java.util.Locale class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { @@ -46,13 +44,12 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { assertEquals(getRes, getRes2) assertISMTemplateEquals(ismTemp, getRes[templateName]) assertISMTemplateEquals(ismTemp2, getRes[templateName2]) - } fun `test get not exist template`() { val tn = "t1" try { - client().makeRequest(GET.toString(), "${ISM_TEMPLATE_BASE_URI}/$tn") + client().makeRequest(GET.toString(), "$ISM_TEMPLATE_BASE_URI/$tn") fail("Expect a failure") } catch (e: ResponseException) { assertEquals("Unexpected RestStatus", RestStatus.NOT_FOUND, e.response.restStatus()) @@ -163,7 +160,6 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { ) assertNull(getManagedIndexConfig(indexName1)) - // hidden index will not be manage assertPredicatesOnMetaData( listOf(indexName1 to listOf(ManagedIndexSettings.POLICY_ID.key to fun(policyID: Any?): Boolean = policyID == null)), @@ -172,5 +168,4 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { ) assertNull(getManagedIndexConfig(indexName3)) } - -} \ No newline at end of file +} From b9aaa7a7ea9dd07d72b6dc58dbe543736d9902c8 Mon Sep 17 00:00:00 2001 From: bowenlan-amzn <bowenlan@amazon.com> Date: Thu, 10 Dec 2020 09:02:34 -0800 Subject: [PATCH 09/21] start to clean code --- .../indexmanagement/IndexManagementPlugin.kt | 38 ++++++------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/IndexManagementPlugin.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/IndexManagementPlugin.kt index 1bf40a949..e9f57759e 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/IndexManagementPlugin.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/IndexManagementPlugin.kt @@ -322,33 +322,19 @@ internal class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, Act ) } - // override fun getNamedXContent(): MutableList<NamedXContentRegistry.Entry> { - // val entries = mutableListOf<NamedXContentRegistry.Entry>() - // val ismTemplateEntry = NamedXContentRegistry.Entry( - // Metadata.Custom::class.java, - // ISMTemplateMetadata.ISM_TEMPLATE, - // ContextParser{ p, _ -> ISMTemplateMetadata.parse(p) } - // ) - // entries.add(ismTemplateEntry) - // return entries - // } - - override fun getNamedWriteables(): MutableList<NamedWriteableRegistry.Entry> { - // ClusterModule 139 - val entries = mutableListOf<NamedWriteableRegistry.Entry>() - val ismTemplateEntry = NamedWriteableRegistry.Entry( - Metadata.Custom::class.java, - ISMTemplateMetadata.TYPE, - Writeable.Reader { sin -> ISMTemplateMetadata(sin) } - ) - val ismTemplateEntry2 = NamedWriteableRegistry.Entry( - NamedDiff::class.java, - ISMTemplateMetadata.TYPE, - Writeable.Reader { sin -> ISMTemplateMetadata.readDiffFrom(sin) } + override fun getNamedWriteables(): List<NamedWriteableRegistry.Entry> { + return listOf( + NamedWriteableRegistry.Entry( + Metadata.Custom::class.java, + ISMTemplateMetadata.TYPE, + Writeable.Reader { sin -> ISMTemplateMetadata(sin) } + ), + NamedWriteableRegistry.Entry( + NamedDiff::class.java, + ISMTemplateMetadata.TYPE, + Writeable.Reader { sin -> ISMTemplateMetadata.readDiffFrom(sin) } + ) ) - entries.add(ismTemplateEntry) - entries.add(ismTemplateEntry2) - return entries } override fun getTransportInterceptors(namedWriteableRegistry: NamedWriteableRegistry, threadContext: ThreadContext): List<TransportInterceptor> { From ff4053547ad5e85ac4b862f1b708cded8ef51c9c Mon Sep 17 00:00:00 2001 From: bowenlan-amzn <bowenlan@amazon.com> Date: Thu, 10 Dec 2020 09:48:42 -0800 Subject: [PATCH 10/21] wanna see code cov --- .codecov.yml | 2 +- .../IndexStateManagementRestTestCase.kt | 7 +++++-- .../resthandler/ISMTemplateRestAPIIT.kt | 9 +++++++++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.codecov.yml b/.codecov.yml index 0e44b28ab..0619cbbb1 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -1,5 +1,5 @@ codecov: - require_ci_to_pass: yes + require_ci_to_pass: no coverage: precision: 2 diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt index 2b4e3f4cf..d5b0d7066 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt @@ -715,12 +715,11 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() name: String, templateString: String ): Response { - val response = client().makeRequest( + return client().makeRequest( "PUT", "$ISM_TEMPLATE_BASE_URI/$name", StringEntity(templateString, APPLICATION_JSON) ) - return response } protected fun getISMTemplatesAsMap(name: String): Map<String, Any> { @@ -811,4 +810,8 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() } return true } + + protected fun deleteISMTemplate(name: String): Response { + return client().makeRequest("DELETE", "$ISM_TEMPLATE_BASE_URI/$name") + } } diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt index e4eb72251..7b14ac0ed 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt @@ -15,6 +15,7 @@ import org.elasticsearch.client.ResponseException import org.elasticsearch.common.settings.Settings import org.elasticsearch.rest.RestStatus import org.elasticsearch.rest.RestRequest.Method.GET +import org.junit.Assert import java.time.Instant import java.time.temporal.ChronoUnit import java.util.Locale @@ -44,6 +45,11 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { assertEquals(getRes, getRes2) assertISMTemplateEquals(ismTemp, getRes[templateName]) assertISMTemplateEquals(ismTemp2, getRes[templateName2]) + + // good to clean up ism template after test + val delRes = deleteISMTemplate(templateName) + deleteISMTemplate(templateName2) + assertEquals(true, delRes.asMap()["acknowledged"]) } fun `test get not exist template`() { @@ -167,5 +173,8 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { true ) assertNull(getManagedIndexConfig(indexName3)) + + // good to clean up ism template after test + deleteISMTemplate(templateName) } } From 182e3c358cd02a2c6fd2652262bb24ba1e669b0f Mon Sep 17 00:00:00 2001 From: bowenlan-amzn <bowenlan@amazon.com> Date: Thu, 10 Dec 2020 10:51:33 -0800 Subject: [PATCH 11/21] clean up 1 --- .codecov.yml | 2 +- .../ISMTemplateService.kt | 87 ++++++++----------- .../ManagedIndexCoordinator.kt | 36 +++++--- .../indexstatemanagement/model/ISMTemplate.kt | 23 +---- .../model/ISMTemplateMetadata.kt | 5 ++ .../resthandler/RestAddISMTemplateAction.kt | 7 -- .../settings/ManagedIndexSettings.kt | 2 +- .../delete/DeleteISMTemplateRequest.kt | 3 + .../TransportDeleteISMTemplateAction.kt | 2 - .../ismtemplate/get/GetISMTemplateRequest.kt | 13 +-- .../ismtemplate/get/GetISMTemplateResponse.kt | 3 + .../get/TransportGetISMTemplateAction.kt | 6 +- .../ismtemplate/put/PutISMTemplateRequest.kt | 20 +++-- .../ismtemplate/put/PutISMTemplateResponse.kt | 3 + .../put/TransportPutISMTemplateAction.kt | 2 - .../IndexStateManagementRestTestCase.kt | 4 - .../resthandler/ISMTemplateRestAPIIT.kt | 55 ++++++------ .../delete/DeleteISMTemplateRequestTests.kt | 2 +- .../get/GetISMTemplateRequestTests.kt | 2 +- .../get/GetISMTemplateResponseTests.kt | 2 +- .../put/PutISMTemplateResponseTests.kt | 2 +- 21 files changed, 133 insertions(+), 148 deletions(-) diff --git a/.codecov.yml b/.codecov.yml index 0619cbbb1..0e44b28ab 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -1,5 +1,5 @@ codecov: - require_ci_to_pass: no + require_ci_to_pass: yes coverage: precision: 2 diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt index a8188cef7..785ffe60b 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt @@ -57,30 +57,27 @@ class ISMTemplateService @Inject constructor( listener: ActionListener<PutISMTemplateResponse> ) { clusterService.submitStateUpdateTask( - IndexManagementPlugin.PLUGIN_NAME, - object : ClusterStateUpdateTask(Priority.NORMAL) { - override fun execute(currentState: ClusterState): ClusterState { - return addISMTemplate(currentState, templateName, template) - } + IndexManagementPlugin.PLUGIN_NAME, + object : ClusterStateUpdateTask(Priority.NORMAL) { + override fun execute(currentState: ClusterState): ClusterState { + return addISMTemplate(currentState, templateName, template) + } - override fun onFailure(source: String, e: Exception) { - listener.onFailure(e) - } + override fun onFailure(source: String, e: Exception) { + listener.onFailure(e) + } - override fun timeout(): TimeValue = masterTimeout + override fun timeout(): TimeValue = masterTimeout - override fun clusterStateProcessed(source: String, oldState: ClusterState, newState: ClusterState) { - log.info("cluster state processed $source") - var status = RestStatus.CREATED - val oldTemplate = oldState.metadata.ismTemplates()[templateName] - if (oldTemplate != null) { - log.info("old template $oldTemplate") - status = RestStatus.OK - } - // oldTemplate != null ?: { status = RestStatus.OK } - listener.onResponse(PutISMTemplateResponse(templateName, template, status)) + override fun clusterStateProcessed(source: String, oldState: ClusterState, newState: ClusterState) { + var status = RestStatus.CREATED + val oldTemplate = oldState.metadata.ismTemplates()[templateName] + if (oldTemplate != null) { + status = RestStatus.OK } + listener.onResponse(PutISMTemplateResponse(templateName, template, status)) } + } ) } @@ -88,14 +85,10 @@ class ISMTemplateService @Inject constructor( val existingTemplates = currentState.metadata.ismTemplates() val existingTemplate = existingTemplates[templateName] - log.info("existing matching template $existingTemplate") - log.info("input template $template") - if (template == existingTemplate) return currentState // find templates with overlapping index pattern val overlaps = findConflictingISMTemplates(templateName, template.indexPatterns, template.priority, existingTemplates) - log.info("find overlapping templates $overlaps") if (overlaps.isNotEmpty()) { val esg = "new ism template $templateName has index pattern ${template.indexPatterns} " + "matching existing templates ${overlaps.entries.stream().map { "${it.key} => ${it.value}" }.collect(Collectors.joining(","))}," + @@ -105,7 +98,6 @@ class ISMTemplateService @Inject constructor( validateFormat(templateName, template.indexPatterns) - log.info("updating ISM template $templateName") return ClusterState.builder(currentState).metadata(Metadata.builder(currentState.metadata()) .putISMTemplate(templateName, template, existingTemplates)).build() } @@ -114,27 +106,25 @@ class ISMTemplateService @Inject constructor( * remove ISM template from cluster state metadata */ fun deleteISMTemplate(templateName: String, masterTimeout: TimeValue, listener: ActionListener<AcknowledgedResponse>) { - log.info("service remove template") clusterService.submitStateUpdateTask( - IndexManagementPlugin.PLUGIN_NAME, - object : ClusterStateUpdateTask(Priority.NORMAL) { - override fun execute(currentState: ClusterState): ClusterState { - log.info("service remove template $templateName") - val existingTemplates = currentState.metadata.ismTemplates() - return ClusterState.builder(currentState).metadata(Metadata.builder(currentState.metadata) - .removeISMTemplate(templateName, existingTemplates)).build() - } + IndexManagementPlugin.PLUGIN_NAME, + object : ClusterStateUpdateTask(Priority.NORMAL) { + override fun execute(currentState: ClusterState): ClusterState { + val existingTemplates = currentState.metadata.ismTemplates() + return ClusterState.builder(currentState).metadata(Metadata.builder(currentState.metadata) + .removeISMTemplate(templateName, existingTemplates)).build() + } - override fun onFailure(source: String, e: Exception) { - listener.onFailure(e) - } + override fun onFailure(source: String, e: Exception) { + listener.onFailure(e) + } - override fun timeout(): TimeValue = masterTimeout + override fun timeout(): TimeValue = masterTimeout - override fun clusterStateProcessed(source: String, oldState: ClusterState, newState: ClusterState) { - listener.onResponse(AcknowledgedResponse(true)) - } + override fun clusterStateProcessed(source: String, oldState: ClusterState, newState: ClusterState) { + listener.onResponse(AcknowledgedResponse(true)) } + } ) } @@ -155,21 +145,19 @@ class ISMTemplateService @Inject constructor( log.info("index $indexName is hidden $isHidden") if (isHidden) return null - val ismTemplates = ismTemplates.filter { (_, template) -> - log.info("index create after template? ${template.lastUpdatedTime.toEpochMilli() < indexMetadata.creationDate}") - template.lastUpdatedTime.toEpochMilli() < indexMetadata.creationDate - } - + // only process indices created after template // traverse all ism templates for matching ones val patternMatchPredicate = { pattern: String -> Regex.simpleMatch(pattern, indexName) } val matchedTemplates = mutableMapOf<ISMTemplate, String>() - ismTemplates.forEach { (templateName, template) -> + ismTemplates.filter { (_, template) -> + log.info("index create after template? ${template.lastUpdatedTime.toEpochMilli() < indexMetadata.creationDate}") + template.lastUpdatedTime.toEpochMilli() < indexMetadata.creationDate + }.forEach { (templateName, template) -> val matched = template.indexPatterns.stream().anyMatch(patternMatchPredicate) if (matched) matchedTemplates[template] = templateName } if (matchedTemplates.isEmpty()) return null - log.info("all matching templates $matchedTemplates") // sort by template priority val winner = matchedTemplates.keys.maxBy { it.priority } @@ -240,11 +228,10 @@ class ISMTemplateService @Inject constructor( priority: Int, ismTemplates: Map<String, ISMTemplate> ): Map<String, List<String>> { - // focus on template with same priority - val ismTemplates = ismTemplates.filter { it.value.priority == priority } val automaton1 = Regex.simpleMatchToAutomaton(*indexPatterns.toTypedArray()) val overlappingTemplates = mutableMapOf<String, List<String>>() - ismTemplates.forEach { (templateName, template) -> + // focus on template with same priority + ismTemplates.filter { it.value.priority == priority }.forEach { (templateName, template) -> val automaton2 = Regex.simpleMatchToAutomaton(*template.indexPatterns.toTypedArray()) if (!Operations.isEmpty(Operations.intersection(automaton1, automaton2))) { log.info("existing template $templateName overlaps candidate $candidate") diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt index eac5db355..78049f1e0 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt @@ -69,7 +69,7 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse import org.elasticsearch.client.Client import org.elasticsearch.cluster.ClusterChangedEvent import org.elasticsearch.cluster.ClusterState -import org.elasticsearch.cluster.LocalNodeMasterListener +import org.elasticsearch.cluster.ClusterStateListener import org.elasticsearch.cluster.service.ClusterService import org.elasticsearch.common.bytes.BytesReference import org.elasticsearch.common.component.LifecycleListener @@ -108,7 +108,7 @@ class ManagedIndexCoordinator( private val clusterService: ClusterService, private val threadPool: ThreadPool, indexManagementIndices: IndexManagementIndices -) : LocalNodeMasterListener, +) : ClusterStateListener, CoroutineScope by CoroutineScope(SupervisorJob() + Dispatchers.Default + CoroutineName("ManagedIndexCoordinator")), LifecycleListener() { @@ -124,10 +124,12 @@ class ManagedIndexCoordinator( BackoffPolicy.constantBackoff(COORDINATOR_BACKOFF_MILLIS.get(settings), COORDINATOR_BACKOFF_COUNT.get(settings)) @Volatile private var jobInterval = JOB_INTERVAL.get(settings) + @Volatile private var isMaster = false + init { clusterService.addListener(this) clusterService.addLifecycleListener(this) - clusterService.addLocalNodeMasterListener(this) + // clusterService.addLocalNodeMasterListener(this) clusterService.clusterSettings.addSettingsUpdateConsumer(SWEEP_PERIOD) { sweepPeriod = it initBackgroundSweep() @@ -144,18 +146,34 @@ class ManagedIndexCoordinator( } } - override fun onMaster() { + private fun executorName(): String { + return ThreadPool.Names.MANAGEMENT + } + + fun onMaster() { // Init background sweep when promoted to being master initBackgroundSweep() } - override fun offMaster() { + fun offMaster() { // Cancel background sweep when demoted from being master scheduledFullSweep?.cancel() } @Suppress("ReturnCount") override fun clusterChanged(event: ClusterChangedEvent) { + // Instead of using a LocalNodeMasterListener to track master changes, this service will + // track them here to avoid conditions where master listener events run after other + // listeners that depend on what happened in the master listener + if (this.isMaster != event.localNodeMaster()) { + this.isMaster = event.localNodeMaster() + if (this.isMaster) { + onMaster() + } else { + offMaster() + } + } + if (!isIndexStateManagementEnabled()) return if (!event.localNodeMaster()) return @@ -230,7 +248,6 @@ class ManagedIndexCoordinator( @OpenForTesting suspend fun sweepClusterChangedEvent(event: ClusterChangedEvent) { - logger.info("start sweep cluster changed event") val indicesDeletedRequests = event.indicesDeleted() .filter { event.previousState().metadata().index(it)?.getPolicyID() != null } .map { deleteManagedIndexRequest(it.uuid) } @@ -288,14 +305,9 @@ class ManagedIndexCoordinator( val indexUuid = indexMetadatas[index].indexUUID val policyID = templates[template]?.policyID if (indexUuid != null && policyID != null) { - logger.info("create request for index $index matching template $template") - logger.info("index name is $index") - logger.info("index uuid is $indexUuid") - logger.info("policy id is $policyID") updateManagedIndexReqs.add(managedIndexConfigIndexRequest(index, indexUuid, policyID, jobInterval)) } } - logger.info("size of matching template req ${updateManagedIndexReqs.size}") return updateManagedIndexReqs } @@ -337,7 +349,7 @@ class ManagedIndexCoordinator( } } - scheduledFullSweep = threadPool.scheduleWithFixedDelay(scheduledSweep, sweepPeriod, ThreadPool.Names.SAME) + scheduledFullSweep = threadPool.scheduleWithFixedDelay(scheduledSweep, sweepPeriod, executorName()) } private fun getFullSweepElapsedTime(): TimeValue = diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplate.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplate.kt index 47955c24d..dcbed958d 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplate.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplate.kt @@ -87,11 +87,9 @@ data class ISMTemplate( var priority = 0 var lastUpdatedTime: Instant? = null - log.info("current token ${xcp.currentToken()}") ensureExpectedToken(Token.START_OBJECT, xcp.nextToken(), xcp) while (xcp.nextToken() != Token.END_OBJECT) { val fieldName = xcp.currentName() - log.info("parse field name $fieldName") xcp.nextToken() when (fieldName) { @@ -99,34 +97,21 @@ data class ISMTemplate( ensureExpectedToken(Token.START_ARRAY, xcp.currentToken(), xcp) while (xcp.nextToken() != Token.END_ARRAY) { indexPatterns.add(xcp.text()) - log.info("field $indexPatterns") } } - POLICY_ID -> { - policyID = xcp.text() - log.info("field $policyID") - } - PRIORITY -> { - priority = if (xcp.currentToken() == Token.VALUE_NULL) 0 else xcp.intValue() - } - LAST_UPDATED_TIME_FIELD -> { - lastUpdatedTime = xcp.instant() - log.info("field last update time $lastUpdatedTime") - } + POLICY_ID -> policyID = xcp.text() + PRIORITY -> priority = if (xcp.currentToken() == Token.VALUE_NULL) 0 else xcp.intValue() + LAST_UPDATED_TIME_FIELD -> lastUpdatedTime = xcp.instant() else -> throw IllegalArgumentException("Invalid field: [$fieldName] found in ISMTemplate.") } } - val result = ISMTemplate( + return ISMTemplate( indexPatterns, requireNotNull(policyID) { "policy id is null" }, priority, lastUpdatedTime ?: Instant.now() ) - - log.info("ism template parse result $result") - // TODO check index pattern is empty or not - return result } fun readISMTemplateDiffFrom(sin: StreamInput): Diff<ISMTemplate> = AbstractDiffable.readDiffFrom(::ISMTemplate, sin) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateMetadata.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateMetadata.kt index 308724c43..67ed7d99a 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateMetadata.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateMetadata.kt @@ -31,6 +31,7 @@ import org.elasticsearch.common.xcontent.ToXContent import org.elasticsearch.common.xcontent.XContentBuilder import org.elasticsearch.common.xcontent.XContentParser import org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken +import java.io.IOException import java.util.EnumSet private val log = LogManager.getLogger(ISMTemplateMetadata::class.java) @@ -42,10 +43,12 @@ private val log = LogManager.getLogger(ISMTemplateMetadata::class.java) // EnrichMetadata class ISMTemplateMetadata(val ismTemplates: Map<String, ISMTemplate>) : Metadata.Custom { + @Throws(IOException::class) constructor(sin: StreamInput) : this( sin.readMap(StreamInput::readString, ::ISMTemplate) ) + @Throws(IOException::class) override fun writeTo(out: StreamOutput) { out.writeMap(ismTemplates, StreamOutput::writeString) { stream, `val` -> `val`.writeTo(stream) } } @@ -78,11 +81,13 @@ class ISMTemplateMetadata(val ismTemplates: Map<String, ISMTemplate>) : Metadata this.ismTemplateDiff = DiffableUtils.diff(before.ismTemplates, after.ismTemplates, DiffableUtils.getStringKeySerializer()) } + @Throws(IOException::class) constructor(sin: StreamInput) { this.ismTemplateDiff = DiffableUtils.readJdkMapDiff(sin, DiffableUtils.getStringKeySerializer(), ::ISMTemplate, ISMTemplate.Companion::readISMTemplateDiffFrom) } + @Throws(IOException::class) override fun writeTo(out: StreamOutput) { ismTemplateDiff.writeTo(out) } diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestAddISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestAddISMTemplateAction.kt index 2582c14c2..fe3774634 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestAddISMTemplateAction.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestAddISMTemplateAction.kt @@ -24,7 +24,6 @@ import org.apache.logging.log4j.LogManager import org.elasticsearch.action.support.master.MasterNodeRequest.DEFAULT_MASTER_NODE_TIMEOUT import org.elasticsearch.client.node.NodeClient import org.elasticsearch.common.xcontent.ToXContent -import org.elasticsearch.common.xcontent.XContentHelper import org.elasticsearch.rest.BaseRestHandler import org.elasticsearch.rest.BytesRestResponse import org.elasticsearch.rest.RestHandler.Route @@ -38,7 +37,6 @@ import java.time.Instant private val log = LogManager.getLogger(RestAddISMTemplateAction::class.java) -// RestIndexPolicyAction class RestAddISMTemplateAction : BaseRestHandler() { override fun routes(): List<Route> { return listOf( @@ -55,13 +53,8 @@ class RestAddISMTemplateAction : BaseRestHandler() { val templateName = request.param("templateID", "") if (templateName == "") { throw IllegalArgumentException("Missing template name") } - log.info("request content ${XContentHelper.convertToMap(request.requiredContent(), false, request.xContentType).v2()}") - val xcp = request.contentParser() - // ISMTemplate.show(xcp) val ismTemplate = ISMTemplate.parse(xcp).copy(lastUpdatedTime = Instant.now()) - log.info("rest template $ismTemplate") - val masterTimeout = request.paramAsTime("master_timeout", DEFAULT_MASTER_NODE_TIMEOUT) val addISMTemplateRequest = PutISMTemplateRequest(templateName, ismTemplate).masterNodeTimeout(masterTimeout) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/settings/ManagedIndexSettings.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/settings/ManagedIndexSettings.kt index a0b67ea6f..a3c9f8621 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/settings/ManagedIndexSettings.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/settings/ManagedIndexSettings.kt @@ -24,7 +24,7 @@ import java.util.function.Function class ManagedIndexSettings { companion object { const val DEFAULT_ISM_ENABLED = true - const val DEFAULT_JOB_INTERVAL = 1 + const val DEFAULT_JOB_INTERVAL = 5 private val ALLOW_LIST_ALL = ActionConfig.ActionType.values().toList().map { it.type } val ALLOW_LIST_NONE = emptyList<String>() val SNAPSHOT_DENY_LIST_NONE = emptyList<String>() diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequest.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequest.kt index 748436224..4a767f862 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequest.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequest.kt @@ -19,6 +19,7 @@ import org.elasticsearch.action.ActionRequestValidationException import org.elasticsearch.action.support.master.MasterNodeRequest import org.elasticsearch.common.io.stream.StreamInput import org.elasticsearch.common.io.stream.StreamOutput +import java.io.IOException class DeleteISMTemplateRequest : MasterNodeRequest<DeleteISMTemplateRequest> { @@ -30,10 +31,12 @@ class DeleteISMTemplateRequest : MasterNodeRequest<DeleteISMTemplateRequest> { this.templateName = templateName } + @Throws(IOException::class) constructor(sin: StreamInput) : super(sin) { templateName = sin.readString() } + @Throws(IOException::class) override fun writeTo(out: StreamOutput) { super.writeTo(out) out.writeString(templateName) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/TransportDeleteISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/TransportDeleteISMTemplateAction.kt index ad79ebb13..3fa4f2b02 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/TransportDeleteISMTemplateAction.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/TransportDeleteISMTemplateAction.kt @@ -21,7 +21,6 @@ import org.elasticsearch.action.ActionListener import org.elasticsearch.action.support.ActionFilters import org.elasticsearch.action.support.master.AcknowledgedResponse import org.elasticsearch.action.support.master.TransportMasterNodeAction -import org.elasticsearch.client.Client import org.elasticsearch.cluster.ClusterState import org.elasticsearch.cluster.block.ClusterBlockException import org.elasticsearch.cluster.block.ClusterBlockLevel @@ -41,7 +40,6 @@ class TransportDeleteISMTemplateAction @Inject constructor( threadPool: ThreadPool, actionFilters: ActionFilters, indexNameExpressionResolver: IndexNameExpressionResolver, - val client: Client, val ismTemplateService: ISMTemplateService ) : TransportMasterNodeAction<DeleteISMTemplateRequest, AcknowledgedResponse>( DeleteISMTemplateAction.NAME, diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequest.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequest.kt index ae506db89..c3e7895ce 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequest.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequest.kt @@ -19,25 +19,28 @@ import org.elasticsearch.action.ActionRequestValidationException import org.elasticsearch.action.support.master.MasterNodeRequest import org.elasticsearch.common.io.stream.StreamInput import org.elasticsearch.common.io.stream.StreamOutput +import java.io.IOException -// GetIndexTemplatesResponse class GetISMTemplateRequest : MasterNodeRequest<GetISMTemplateRequest> { + // TODO not sure array is the right choice val templateNames: Array<String> + constructor(templateName: Array<String>) : super() { + this.templateNames = templateName + } + + @Throws(IOException::class) constructor(sin: StreamInput) : super(sin) { templateNames = sin.readStringArray() } + @Throws(IOException::class) override fun writeTo(out: StreamOutput) { super.writeTo(out) out.writeStringArray(templateNames) } - constructor(templateName: Array<String>) : super() { - this.templateNames = templateName - } - override fun validate(): ActionRequestValidationException? { return null } diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponse.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponse.kt index e28249746..628e696b1 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponse.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponse.kt @@ -7,6 +7,7 @@ import org.elasticsearch.common.io.stream.StreamOutput import org.elasticsearch.common.xcontent.ToXContent import org.elasticsearch.common.xcontent.ToXContentObject import org.elasticsearch.common.xcontent.XContentBuilder +import java.io.IOException // GetComposableIndexTemplateAction.Response class GetISMTemplateResponse : ActionResponse, ToXContentObject { @@ -17,6 +18,7 @@ class GetISMTemplateResponse : ActionResponse, ToXContentObject { this.ismTemplates = ismTemplates } + @Throws(IOException::class) constructor(sin: StreamInput) : super(sin) { val size = sin.readVInt() ismTemplates = mutableMapOf() @@ -25,6 +27,7 @@ class GetISMTemplateResponse : ActionResponse, ToXContentObject { } } + @Throws(IOException::class) override fun writeTo(out: StreamOutput) { out.writeVInt(ismTemplates.size) ismTemplates.forEach { (k, v) -> diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/TransportGetISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/TransportGetISMTemplateAction.kt index f28d9a472..7a811aefc 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/TransportGetISMTemplateAction.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/TransportGetISMTemplateAction.kt @@ -1,6 +1,5 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ISMTemplateService import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.ismTemplates import org.apache.logging.log4j.LogManager @@ -8,7 +7,6 @@ import org.elasticsearch.ResourceNotFoundException import org.elasticsearch.action.ActionListener import org.elasticsearch.action.support.ActionFilters import org.elasticsearch.action.support.master.TransportMasterNodeAction -import org.elasticsearch.client.Client import org.elasticsearch.cluster.ClusterState import org.elasticsearch.cluster.block.ClusterBlockException import org.elasticsearch.cluster.block.ClusterBlockLevel @@ -29,9 +27,7 @@ class TransportGetISMTemplateAction @Inject constructor( clusterService: ClusterService, threadPool: ThreadPool, actionFilters: ActionFilters, - indexNameExpressionResolver: IndexNameExpressionResolver, - val client: Client, - val ismTemplateService: ISMTemplateService + indexNameExpressionResolver: IndexNameExpressionResolver ) : TransportMasterNodeAction<GetISMTemplateRequest, GetISMTemplateResponse>( GetISMTemplateAction.NAME, transportService, diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateRequest.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateRequest.kt index 53b33866a..64b6ea0ae 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateRequest.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateRequest.kt @@ -20,32 +20,34 @@ import org.elasticsearch.action.ActionRequestValidationException import org.elasticsearch.action.support.master.MasterNodeRequest import org.elasticsearch.common.io.stream.StreamInput import org.elasticsearch.common.io.stream.StreamOutput +import java.io.IOException -// PutComponentTemplateAction class PutISMTemplateRequest : MasterNodeRequest<PutISMTemplateRequest> { val templateName: String val ismTemplate: ISMTemplate + constructor( + templateName: String, + ismTemplate: ISMTemplate + ) : super() { + this.templateName = templateName + this.ismTemplate = ismTemplate + } + + @Throws(IOException::class) constructor(sin: StreamInput) : super(sin) { templateName = sin.readString() ismTemplate = ISMTemplate(sin) } + @Throws(IOException::class) override fun writeTo(out: StreamOutput) { super.writeTo(out) out.writeString(templateName) ismTemplate.writeTo(out) } - constructor( - templateName: String, - ismTemplate: ISMTemplate - ) : super() { - this.templateName = templateName - this.ismTemplate = ismTemplate - } - override fun validate(): ActionRequestValidationException? { return null } diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponse.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponse.kt index e63915c03..e3fd71411 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponse.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponse.kt @@ -25,6 +25,7 @@ import org.elasticsearch.common.xcontent.ToXContent import org.elasticsearch.common.xcontent.ToXContentObject import org.elasticsearch.common.xcontent.XContentBuilder import org.elasticsearch.rest.RestStatus +import java.io.IOException class PutISMTemplateResponse : ActionResponse, ToXContentObject { @@ -42,12 +43,14 @@ class PutISMTemplateResponse : ActionResponse, ToXContentObject { this.status = status } + @Throws(IOException::class) constructor(sin: StreamInput) : this( id = sin.readString(), template = ISMTemplate(sin), status = sin.readEnum(RestStatus::class.java) ) + @Throws(IOException::class) override fun writeTo(out: StreamOutput) { out.writeString(id) template.writeTo(out) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/TransportPutISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/TransportPutISMTemplateAction.kt index bfac1e6bc..066328fd1 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/TransportPutISMTemplateAction.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/TransportPutISMTemplateAction.kt @@ -20,7 +20,6 @@ import org.apache.logging.log4j.LogManager import org.elasticsearch.action.ActionListener import org.elasticsearch.action.support.ActionFilters import org.elasticsearch.action.support.master.TransportMasterNodeAction -import org.elasticsearch.client.Client import org.elasticsearch.cluster.ClusterState import org.elasticsearch.cluster.block.ClusterBlockException import org.elasticsearch.cluster.block.ClusterBlockLevel @@ -40,7 +39,6 @@ class TransportPutISMTemplateAction @Inject constructor( threadPool: ThreadPool, actionFilters: ActionFilters, indexNameExpressionResolver: IndexNameExpressionResolver, - val client: Client, val ismTemplateService: ISMTemplateService ) : TransportMasterNodeAction<PutISMTemplateRequest, PutISMTemplateResponse>( PutISMTemplateAction.NAME, diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt index d5b0d7066..1b5a44ec5 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt @@ -479,7 +479,6 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() metadata = ManagedIndexMetaData.parse(xcp) } - println("get back metadata is $metadata") return metadata } @@ -751,13 +750,10 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() var template: ISMTemplate? = null while (xcp.nextToken() != Token.END_ARRAY) { - println("t current name: ${xcp.currentName()}") - println("t current token: ${xcp.currentToken()}") when (xcp.currentName()) { TEMPLATE_NAME -> { xcp.nextToken() templateName = xcp.text() - println("template name $templateName") } ISM_TEMPLATE -> { // xcp.nextToken() diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt index 7b14ac0ed..1c2287a7c 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt @@ -15,7 +15,7 @@ import org.elasticsearch.client.ResponseException import org.elasticsearch.common.settings.Settings import org.elasticsearch.rest.RestStatus import org.elasticsearch.rest.RestRequest.Method.GET -import org.junit.Assert +import org.junit.After import java.time.Instant import java.time.temporal.ChronoUnit import java.util.Locale @@ -24,8 +24,16 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { private val testIndexName = javaClass.simpleName.toLowerCase(Locale.ROOT) + private val templateName = "t1" + private val templateName2 = "t2" + + @After + fun `clean template`() { + deleteISMTemplate(templateName) + deleteISMTemplate(templateName2) + } + fun `test ISM template`() { - val templateName = "t1" val ismTemp = ISMTemplate(listOf("log*"), "policy_1", 100, randomInstant()) var res = createISMTemplate(templateName, ismTemp) @@ -37,7 +45,6 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { var getRes = getISMTemplatesAsObject(templateName) assertISMTemplateEquals(ismTemp, getRes[templateName]) - val templateName2 = "t2" val ismTemp2 = ISMTemplate(listOf("trace*"), "policy_1", 100, randomInstant()) createISMTemplate(templateName2, ismTemp2) getRes = getISMTemplatesAsObject("$templateName,$templateName2") @@ -46,16 +53,13 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { assertISMTemplateEquals(ismTemp, getRes[templateName]) assertISMTemplateEquals(ismTemp2, getRes[templateName2]) - // good to clean up ism template after test val delRes = deleteISMTemplate(templateName) - deleteISMTemplate(templateName2) assertEquals(true, delRes.asMap()["acknowledged"]) } fun `test get not exist template`() { - val tn = "t1" try { - client().makeRequest(GET.toString(), "$ISM_TEMPLATE_BASE_URI/$tn") + client().makeRequest(GET.toString(), "$ISM_TEMPLATE_BASE_URI/$templateName") fail("Expect a failure") } catch (e: ResponseException) { assertEquals("Unexpected RestStatus", RestStatus.NOT_FOUND, e.response.restStatus()) @@ -63,10 +67,10 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { val expectErrorMessage = mapOf( "error" to mapOf( "root_cause" to listOf<Map<String, Any>>( - mapOf("type" to "resource_not_found_exception", "reason" to "index template matching [$tn] not found") + mapOf("type" to "resource_not_found_exception", "reason" to "index template matching [$templateName] not found") ), "type" to "resource_not_found_exception", - "reason" to "index template matching [$tn] not found" + "reason" to "index template matching [$templateName] not found" ), "status" to 404 ) @@ -75,20 +79,19 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { } fun `test add template with invalid index pattern`() { - val tn = "t1" try { val ismTemp = ISMTemplate(listOf(" "), "policy_1", 100, randomInstant()) - createISMTemplate(tn, ismTemp) + createISMTemplate(templateName, ismTemp) fail("Expect a failure") } catch (e: ResponseException) { assertEquals("Unexpected RestStatus", RestStatus.BAD_REQUEST, e.response.restStatus()) val actualMessage = e.response.asMap() val expectErrorMessage = mapOf( "error" to mapOf( - "reason" to "index_template [$tn] invalid, cause [Validation Failed: 1: index_patterns [ ] must not contain a space;2: index_pattern [ ] must not contain the following characters [ , \", *, \\, <, |, ,, >, /, ?];]", + "reason" to "index_template [$templateName] invalid, cause [Validation Failed: 1: index_patterns [ ] must not contain a space;2: index_pattern [ ] must not contain the following characters [ , \", *, \\, <, |, ,, >, /, ?];]", "type" to "invalid_index_template_exception", "root_cause" to listOf<Map<String, Any>>( - mapOf("reason" to "index_template [$tn] invalid, cause [Validation Failed: 1: index_patterns [ ] must not contain a space;2: index_pattern [ ] must not contain the following characters [ , \", *, \\, <, |, ,, >, /, ?];]", + mapOf("reason" to "index_template [$templateName] invalid, cause [Validation Failed: 1: index_patterns [ ] must not contain a space;2: index_pattern [ ] must not contain the following characters [ , \", *, \\, <, |, ,, >, /, ?];]", "type" to "invalid_index_template_exception") ) ), @@ -99,21 +102,20 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { } fun `test add template with overlapping index pattern`() { - val tn = "t2" try { val ismTemp = ISMTemplate(listOf("log*"), "policy_1", 100, randomInstant()) - createISMTemplate("t1", ismTemp) - createISMTemplate(tn, ismTemp) + createISMTemplate(templateName, ismTemp) + createISMTemplate(templateName2, ismTemp) fail("Expect a failure") } catch (e: ResponseException) { assertEquals("Unexpected RestStatus", RestStatus.BAD_REQUEST, e.response.restStatus()) val actualMessage = e.response.asMap() val expectErrorMessage = mapOf( "error" to mapOf( - "reason" to "new ism template $tn has index pattern [log*] matching existing templates t1 => [log*], please use a different priority than 100", + "reason" to "new ism template $templateName2 has index pattern [log*] matching existing templates t1 => [log*], please use a different priority than 100", "type" to "illegal_argument_exception", "root_cause" to listOf<Map<String, Any>>( - mapOf("reason" to "new ism template $tn has index pattern [log*] matching existing templates t1 => [log*], please use a different priority than 100", + mapOf("reason" to "new ism template $templateName2 has index pattern [log*] matching existing templates t1 => [log*], please use a different priority than 100", "type" to "illegal_argument_exception") ) ), @@ -131,12 +133,10 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { // need to specify policyID null, can remove after policyID deprecated createIndex(indexName1, null) - val templateName = "t1" + val ismTemp = ISMTemplate(listOf("log*"), policyID, 100, randomInstant()) createISMTemplate(templateName, ismTemp) - println("ism template: ${getISMTemplatesAsObject(null)}") - val actionConfig = ReadOnlyActionConfig(0) val states = listOf( State("ReadOnlyState", listOf(actionConfig), listOf()) @@ -151,12 +151,16 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { states = states ) createPolicy(policy, policyID) + createIndex(indexName2, null) createIndex(indexName3, Settings.builder().put("index.hidden", true).build()) - val managedIndexConfig = getExistingManagedIndexConfig(indexName2) - updateManagedIndexConfigStartTime(managedIndexConfig) - waitFor { assertEquals(policyID, getExplainManagedIndexMetaData(indexName2).policyID) } + waitFor { assertNotNull(getManagedIndexConfig(indexName2)) } + + // TODO uncomment in remove policy id + // val managedIndexConfig = getExistingManagedIndexConfig(indexName2) + // updateManagedIndexConfigStartTime(managedIndexConfig) + // waitFor { assertEquals(policyID, getExplainManagedIndexMetaData(indexName2).policyID) } // only index create after template can be managed assertPredicatesOnMetaData( @@ -173,8 +177,5 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { true ) assertNull(getManagedIndexConfig(indexName3)) - - // good to clean up ism template after test - deleteISMTemplate(templateName) } } diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequestTests.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequestTests.kt index 78759c002..260cae0ee 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequestTests.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequestTests.kt @@ -31,4 +31,4 @@ class DeleteISMTemplateRequestTests : ESTestCase() { val newReq = DeleteISMTemplateRequest(sin) assertEquals(templateName, newReq.templateName) } -} \ No newline at end of file +} diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequestTests.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequestTests.kt index 5e46e3732..68a8e3721 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequestTests.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequestTests.kt @@ -31,4 +31,4 @@ class GetISMTemplateRequestTests : ESTestCase() { val newReq = GetISMTemplateRequest(sin) assertEquals(templateNames, newReq.templateNames) } -} \ No newline at end of file +} diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponseTests.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponseTests.kt index a9ca3c2ad..e55ad8ec8 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponseTests.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponseTests.kt @@ -33,4 +33,4 @@ class GetISMTemplateResponseTests : ESTestCase() { val newReq = GetISMTemplateResponse(sin) assertEquals(ismTemplates, newReq.ismTemplates) } -} \ No newline at end of file +} diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponseTests.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponseTests.kt index 031923c5a..3d9f25c97 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponseTests.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponseTests.kt @@ -38,4 +38,4 @@ class PutISMTemplateResponseTests : ESTestCase() { assertEquals(template, newReq.template) assertEquals(status, newReq.status) } -} \ No newline at end of file +} From 27b1f6f2aeb0048096d3603cc7e57a7d2a897e4b Mon Sep 17 00:00:00 2001 From: bowenlan-amzn <bowenlan@amazon.com> Date: Thu, 10 Dec 2020 17:05:35 -0800 Subject: [PATCH 12/21] try remove seeming not used part in template metadata --- .../model/ISMTemplateMetadata.kt | 37 +------------------ 1 file changed, 1 insertion(+), 36 deletions(-) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateMetadata.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateMetadata.kt index 67ed7d99a..bc39aae64 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateMetadata.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateMetadata.kt @@ -24,13 +24,8 @@ import org.elasticsearch.cluster.metadata.Metadata import org.elasticsearch.common.ParseField import org.elasticsearch.common.io.stream.StreamInput import org.elasticsearch.common.io.stream.StreamOutput -import org.elasticsearch.common.xcontent.ConstructingObjectParser -import org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg -import org.elasticsearch.common.xcontent.ContextParser import org.elasticsearch.common.xcontent.ToXContent import org.elasticsearch.common.xcontent.XContentBuilder -import org.elasticsearch.common.xcontent.XContentParser -import org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken import java.io.IOException import java.util.EnumSet @@ -58,7 +53,6 @@ class ISMTemplateMetadata(val ismTemplates: Map<String, ISMTemplate>) : Metadata } override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - log.info("ism template metadata toXContent: $ismTemplates") builder.startObject(ISM_TEMPLATE.preferredName) ismTemplates.forEach { (k, v) -> builder.field(k, v) @@ -84,7 +78,7 @@ class ISMTemplateMetadata(val ismTemplates: Map<String, ISMTemplate>) : Metadata @Throws(IOException::class) constructor(sin: StreamInput) { this.ismTemplateDiff = DiffableUtils.readJdkMapDiff(sin, DiffableUtils.getStringKeySerializer(), - ::ISMTemplate, ISMTemplate.Companion::readISMTemplateDiffFrom) + ::ISMTemplate, ISMTemplate.Companion::readISMTemplateDiffFrom) } @Throws(IOException::class) @@ -103,35 +97,6 @@ class ISMTemplateMetadata(val ismTemplates: Map<String, ISMTemplate>) : Metadata val TYPE = "ism_template" val ISM_TEMPLATE = ParseField("ism_template") - val PARSER = ConstructingObjectParser<ISMTemplateMetadata, Void>(TYPE, false - ) { a -> ISMTemplateMetadata(a[0] as Map<String, ISMTemplate>) } - - init { - PARSER.declareObject(constructorArg(), ContextParser<Void, Map<String, ISMTemplate>> { p, _ -> - val templates = mutableMapOf<String, ISMTemplate>() - while (p.nextToken() != XContentParser.Token.END_OBJECT) { - val name = p.currentName() - templates[name] = ISMTemplate.parse(p) - } - templates - }, ISM_TEMPLATE) - } - - fun fromStreamInput(sin: StreamInput) = ISMTemplateMetadata(sin.readMap(StreamInput::readString, ::ISMTemplate)) - - // fun parse(xcp: XContentParser): ISMTemplateMetadata = PARSER.parse(xcp, null) - fun parse(xcp: XContentParser): ISMTemplateMetadata { - val ismTemplates = mutableMapOf<String, ISMTemplate>() - log.info("ism template metadata parse, first token ${xcp.currentToken()}") - ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp) - while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { - val fieldName = xcp.currentName() - log.info("current field name $fieldName") - ismTemplates[fieldName] = ISMTemplate.parse(xcp) - } - return ISMTemplateMetadata(ismTemplates) - } - fun readDiffFrom(sin: StreamInput): NamedDiff<Metadata.Custom> = ISMTemplateMetadataDiff(sin) } } From 82f368a3965f0814b8010e690f74673837c6643f Mon Sep 17 00:00:00 2001 From: bowenlan-amzn <bowenlan@amazon.com> Date: Wed, 16 Dec 2020 15:38:57 -0800 Subject: [PATCH 13/21] clean up --- .../indexstatemanagement/ISMTemplateService.kt | 17 ++++++++++++----- .../ManagedIndexCoordinator.kt | 11 +++++------ 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt index 785ffe60b..336a0ee70 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt @@ -38,12 +38,11 @@ import org.elasticsearch.common.regex.Regex import org.elasticsearch.common.unit.TimeValue import org.elasticsearch.indices.InvalidIndexTemplateException import org.elasticsearch.rest.RestStatus -import java.util.* import java.util.stream.Collectors +import java.util.Locale private val log = LogManager.getLogger(ISMTemplateService::class.java) -// MetadataIndexTemplateService class ISMTemplateService @Inject constructor( val clusterService: ClusterService ) { @@ -130,12 +129,15 @@ class ISMTemplateService @Inject constructor( companion object { /** - * find the matching template name for the given index name + * find the matching template for the index * * filter out hidden index * filter out older index than template lastUpdateTime + * + * @param ismTemplates current ISM templates saved in metadata + * @param indexMetadata cluster state index metadata + * @return template name matching with given index */ - // findV2Template @Suppress("ReturnCount") fun findMatchingISMTemplate(ismTemplates: Map<String, ISMTemplate>, indexMetadata: IndexMetadata): String? { val indexName = indexMetadata.index.name @@ -165,6 +167,10 @@ class ISMTemplateService @Inject constructor( return matchedTemplates[winner] } + /** + * validate the template Name and indexPattern provided in the template + * reusing ES validate function in MetadataIndexTemplateService + */ @Suppress("ComplexMethod") fun validateFormat(templateName: String, indexPatterns: List<String>) { val validationErrors = mutableListOf<String>() @@ -220,7 +226,6 @@ class ISMTemplateService @Inject constructor( * * @return map of overlapping template name to its index patterns */ - // addIndexTemplateV2 findConflictingV2Templates @Suppress("SpreadOperator") fun findConflictingISMTemplates( candidate: String, @@ -230,6 +235,7 @@ class ISMTemplateService @Inject constructor( ): Map<String, List<String>> { val automaton1 = Regex.simpleMatchToAutomaton(*indexPatterns.toTypedArray()) val overlappingTemplates = mutableMapOf<String, List<String>>() + // focus on template with same priority ismTemplates.filter { it.value.priority == priority }.forEach { (templateName, template) -> val automaton2 = Regex.simpleMatchToAutomaton(*template.indexPatterns.toTypedArray()) @@ -239,6 +245,7 @@ class ISMTemplateService @Inject constructor( } } overlappingTemplates.remove(candidate) + return overlappingTemplates } } diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt index 78049f1e0..ba6a477dd 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt @@ -129,7 +129,6 @@ class ManagedIndexCoordinator( init { clusterService.addListener(this) clusterService.addLifecycleListener(this) - // clusterService.addLocalNodeMasterListener(this) clusterService.clusterSettings.addSettingsUpdateConsumer(SWEEP_PERIOD) { sweepPeriod = it initBackgroundSweep() @@ -281,7 +280,7 @@ class ManagedIndexCoordinator( if (it.value.shouldDeleteManagedIndexMetaData()) indicesToRemoveManagedIndexMetaDataFrom.add(it.value.index) } - // check newly created indices matching any ISM templates + // check if newly created indices matching any ISM templates val updateMatchingIndexReqs = getMatchingIndicesUpdateReqs(event.state(), event.indicesCreated()) if (updateMatchingIndexReqs.isNotEmpty()) hasCreateRequests = true @@ -290,18 +289,18 @@ class ManagedIndexCoordinator( } /** - * Get update requests for indices matching any ISM templates + * build requests to create jobs for indices matching ISM templates */ fun getMatchingIndicesUpdateReqs(clusterState: ClusterState, indexNames: List<String>): List<DocWriteRequest<*>> { val indexMetadatas = clusterState.metadata.indices val templates = clusterState.metadata.ismTemplates() - val matchingTemplates = indexNames.map { indexName -> + val indexToMatchMap = indexNames.map { indexName -> indexName to findMatchingISMTemplate(templates, indexMetadatas[indexName]) }.toMap() val updateManagedIndexReqs = mutableListOf<DocWriteRequest<*>>() - matchingTemplates.filter { (_, template) -> template != null }.forEach { (index, template) -> + indexToMatchMap.filter { (_, template) -> template != null }.forEach { (index, template) -> val indexUuid = indexMetadatas[index].indexUUID val policyID = templates[template]?.policyID if (indexUuid != null && policyID != null) { @@ -365,7 +364,7 @@ class ManagedIndexCoordinator( suspend fun sweep() { val currentManagedIndices = sweepManagedIndexJobs(client, ismIndices.indexManagementIndexExists()) - // check all un-managed indices, if its name matches any template and older than that template + // check all un-managed indices, if matches any ism template val unManagedIndices = clusterService.state().metadata.indices.values().filterNotNull() .filter { it.value.indexUUID !in currentManagedIndices.keys }.map { it.value.index.name } val updateMatchingIndicesReqs = getMatchingIndicesUpdateReqs(clusterService.state(), unManagedIndices) From 24a47915654a74c007ef6d6def92475ab3454cf3 Mon Sep 17 00:00:00 2001 From: bowenlan-amzn <bowenlan@amazon.com> Date: Mon, 11 Jan 2021 19:58:25 -0800 Subject: [PATCH 14/21] going to clean up --- .../indexmanagement/IndexManagementPlugin.kt | 39 +---- .../elasticapi/ElasticExtensions.kt | 9 ++ .../ISMTemplateService.kt | 140 ++---------------- .../ManagedIndexCoordinator.kt | 44 +++++- .../elasticapi/ElasticExtensions.kt | 31 ++++ .../indexstatemanagement/model/ISMTemplate.kt | 26 +--- .../model/ISMTemplateMetadata.kt | 102 ------------- .../indexstatemanagement/model/Policy.kt | 18 ++- .../resthandler/RestAddISMTemplateAction.kt | 74 --------- .../RestDeleteISMTemplateAction.kt | 53 ------- .../resthandler/RestGetISMTemplateAction.kt | 54 ------- .../indexpolicy/TransportIndexPolicyAction.kt | 56 ++++++- .../delete/DeleteISMTemplateAction.kt | 26 ---- .../delete/DeleteISMTemplateRequest.kt | 48 ------ .../TransportDeleteISMTemplateAction.kt | 68 --------- .../ismtemplate/get/GetISMTemplateAction.kt | 25 ---- .../ismtemplate/get/GetISMTemplateRequest.kt | 47 ------ .../ismtemplate/get/GetISMTemplateResponse.kt | 52 ------- .../get/TransportGetISMTemplateAction.kt | 74 --------- .../ismtemplate/put/PutISMTemplateAction.kt | 25 ---- .../ismtemplate/put/PutISMTemplateRequest.kt | 54 ------- .../ismtemplate/put/PutISMTemplateResponse.kt | 66 --------- .../put/TransportPutISMTemplateAction.kt | 72 --------- .../util/RestHandlerUtils.kt | 23 +-- .../util/IndexManagementException.kt | 45 ++++++ .../mappings/opendistro-ism-config.json | 17 +++ .../IndexStateManagementRestTestCase.kt | 80 ---------- .../indexstatemanagement/TestHelpers.kt | 9 +- .../resthandler/ISMTemplateRestAPIIT.kt | 108 +++----------- .../transport/action/ActionTests.kt | 18 --- .../delete/DeleteISMTemplateRequestTests.kt | 34 ----- .../get/GetISMTemplateRequestTests.kt | 34 ----- .../get/GetISMTemplateResponseTests.kt | 36 ----- .../put/PutISMTemplateRequestTests.kt | 38 ----- .../put/PutISMTemplateResponseTests.kt | 41 ----- .../cached-opendistro-ism-config.json | 17 +++ 36 files changed, 265 insertions(+), 1438 deletions(-) delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateMetadata.kt delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestAddISMTemplateAction.kt delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestDeleteISMTemplateAction.kt delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestGetISMTemplateAction.kt delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateAction.kt delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequest.kt delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/TransportDeleteISMTemplateAction.kt delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateAction.kt delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequest.kt delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponse.kt delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/TransportGetISMTemplateAction.kt delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateAction.kt delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateRequest.kt delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponse.kt delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/TransportPutISMTemplateAction.kt create mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/util/IndexManagementException.kt delete mode 100644 src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequestTests.kt delete mode 100644 src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequestTests.kt delete mode 100644 src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponseTests.kt delete mode 100644 src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateRequestTests.kt delete mode 100644 src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponseTests.kt diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/IndexManagementPlugin.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/IndexManagementPlugin.kt index e9f57759e..ae31e6055 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/IndexManagementPlugin.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/IndexManagementPlugin.kt @@ -18,18 +18,14 @@ package com.amazon.opendistroforelasticsearch.indexmanagement import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.IndexStateManagementHistory import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ManagedIndexCoordinator import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ManagedIndexRunner -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplateMetadata import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.updateindexmetadata.TransportUpdateManagedIndexMetaDataAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.updateindexmetadata.UpdateManagedIndexMetaDataAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.Policy -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestAddISMTemplateAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestAddPolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestChangePolicyAction -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestDeleteISMTemplateAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestDeletePolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestExplainAction -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestGetISMTemplateAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestGetPolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestIndexPolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestRemovePolicyAction @@ -47,12 +43,6 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagemen import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.getpolicy.TransportGetPolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.indexpolicy.IndexPolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.indexpolicy.TransportIndexPolicyAction -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.delete.DeleteISMTemplateAction -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.delete.TransportDeleteISMTemplateAction -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get.GetISMTemplateAction -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get.TransportGetISMTemplateAction -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put.PutISMTemplateAction -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put.TransportPutISMTemplateAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.removepolicy.RemovePolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.removepolicy.TransportRemovePolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.retryfailedmanagedindex.RetryFailedManagedIndexAction @@ -100,13 +90,10 @@ import org.elasticsearch.action.ActionRequest import org.elasticsearch.action.ActionResponse import org.elasticsearch.action.support.ActionFilter import org.elasticsearch.client.Client -import org.elasticsearch.cluster.NamedDiff import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver -import org.elasticsearch.cluster.metadata.Metadata import org.elasticsearch.cluster.node.DiscoveryNodes import org.elasticsearch.cluster.service.ClusterService import org.elasticsearch.common.io.stream.NamedWriteableRegistry -import org.elasticsearch.common.io.stream.Writeable import org.elasticsearch.common.settings.ClusterSettings import org.elasticsearch.common.settings.IndexScopedSettings import org.elasticsearch.common.settings.Setting @@ -145,7 +132,6 @@ internal class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, Act const val ROLLUP_BASE_URI = "$OPEN_DISTRO_BASE_URI/_rollup" const val POLICY_BASE_URI = "$ISM_BASE_URI/policies" const val ROLLUP_JOBS_BASE_URI = "$ROLLUP_BASE_URI/jobs" - const val ISM_TEMPLATE_BASE_URI = "$ISM_BASE_URI/templates" const val INDEX_MANAGEMENT_INDEX = ".opendistro-ism-config" const val INDEX_MANAGEMENT_JOB_TYPE = "opendistro-index-management" const val INDEX_STATE_MANAGEMENT_HISTORY_TYPE = "managed_index_meta_data" @@ -211,10 +197,7 @@ internal class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, Act RestIndexRollupAction(), RestStartRollupAction(), RestStopRollupAction(), - RestExplainRollupAction(), - RestAddISMTemplateAction(), - RestGetISMTemplateAction(), - RestDeleteISMTemplateAction() + RestExplainRollupAction() ) } @@ -315,25 +298,7 @@ internal class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, Act ActionPlugin.ActionHandler(StartRollupAction.INSTANCE, TransportStartRollupAction::class.java), ActionPlugin.ActionHandler(StopRollupAction.INSTANCE, TransportStopRollupAction::class.java), ActionPlugin.ActionHandler(ExplainRollupAction.INSTANCE, TransportExplainRollupAction::class.java), - ActionPlugin.ActionHandler(UpdateRollupMappingAction.INSTANCE, TransportUpdateRollupMappingAction::class.java), - ActionPlugin.ActionHandler(PutISMTemplateAction.INSTANCE, TransportPutISMTemplateAction::class.java), - ActionPlugin.ActionHandler(GetISMTemplateAction.INSTANCE, TransportGetISMTemplateAction::class.java), - ActionPlugin.ActionHandler(DeleteISMTemplateAction.INSTANCE, TransportDeleteISMTemplateAction::class.java) - ) - } - - override fun getNamedWriteables(): List<NamedWriteableRegistry.Entry> { - return listOf( - NamedWriteableRegistry.Entry( - Metadata.Custom::class.java, - ISMTemplateMetadata.TYPE, - Writeable.Reader { sin -> ISMTemplateMetadata(sin) } - ), - NamedWriteableRegistry.Entry( - NamedDiff::class.java, - ISMTemplateMetadata.TYPE, - Writeable.Reader { sin -> ISMTemplateMetadata.readDiffFrom(sin) } - ) + ActionPlugin.ActionHandler(UpdateRollupMappingAction.INSTANCE, TransportUpdateRollupMappingAction::class.java) ) } diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/elasticapi/ElasticExtensions.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/elasticapi/ElasticExtensions.kt index d0dab01d2..735ba75ca 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/elasticapi/ElasticExtensions.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/elasticapi/ElasticExtensions.kt @@ -17,6 +17,8 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.Policy import com.amazon.opendistroforelasticsearch.indexmanagement.util.NO_ID import com.amazon.opendistroforelasticsearch.jobscheduler.spi.utils.LockService import kotlinx.coroutines.delay @@ -70,6 +72,13 @@ fun XContentBuilder.optionalTimeField(name: String, instant: Instant?): XContent return this.timeField(name, "${name}_in_millis", instant.toEpochMilli()) } +fun XContentBuilder.optionalISMTemplateField(name: String, ismTemplate: ISMTemplate?): XContentBuilder { + if (ismTemplate == null) { + return nullField(name) + } + return this.field(Policy.ISM_TEMPLATE, ismTemplate) +} + /** * Retries the given [block] of code as specified by the receiver [BackoffPolicy], * if [block] throws an [ElasticsearchException] that is retriable (502, 503, 504). diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt index 336a0ee70..a8d5849f7 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt @@ -15,118 +15,20 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement -import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.filterNotNullValues import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put.PutISMTemplateResponse -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.ismTemplates -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.putISMTemplate -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.removeISMTemplate +import com.amazon.opendistroforelasticsearch.indexmanagement.util.IndexManagementException import org.apache.logging.log4j.LogManager import org.apache.lucene.util.automaton.Operations -import org.elasticsearch.action.ActionListener -import org.elasticsearch.action.support.master.AcknowledgedResponse -import org.elasticsearch.cluster.ClusterState -import org.elasticsearch.cluster.ClusterStateUpdateTask +import org.elasticsearch.ElasticsearchException import org.elasticsearch.cluster.metadata.IndexMetadata -import org.elasticsearch.cluster.metadata.Metadata -import org.elasticsearch.cluster.service.ClusterService -import org.elasticsearch.common.Priority import org.elasticsearch.common.Strings import org.elasticsearch.common.ValidationException -import org.elasticsearch.common.inject.Inject import org.elasticsearch.common.regex.Regex -import org.elasticsearch.common.unit.TimeValue -import org.elasticsearch.indices.InvalidIndexTemplateException -import org.elasticsearch.rest.RestStatus -import java.util.stream.Collectors -import java.util.Locale private val log = LogManager.getLogger(ISMTemplateService::class.java) -class ISMTemplateService @Inject constructor( - val clusterService: ClusterService -) { - /** - * save ISM template to cluster state metadata - */ - fun putISMTemplate( - templateName: String, - template: ISMTemplate, - masterTimeout: TimeValue, - listener: ActionListener<PutISMTemplateResponse> - ) { - clusterService.submitStateUpdateTask( - IndexManagementPlugin.PLUGIN_NAME, - object : ClusterStateUpdateTask(Priority.NORMAL) { - override fun execute(currentState: ClusterState): ClusterState { - return addISMTemplate(currentState, templateName, template) - } - - override fun onFailure(source: String, e: Exception) { - listener.onFailure(e) - } - - override fun timeout(): TimeValue = masterTimeout - - override fun clusterStateProcessed(source: String, oldState: ClusterState, newState: ClusterState) { - var status = RestStatus.CREATED - val oldTemplate = oldState.metadata.ismTemplates()[templateName] - if (oldTemplate != null) { - status = RestStatus.OK - } - listener.onResponse(PutISMTemplateResponse(templateName, template, status)) - } - } - ) - } - - fun addISMTemplate(currentState: ClusterState, templateName: String, template: ISMTemplate): ClusterState { - val existingTemplates = currentState.metadata.ismTemplates() - val existingTemplate = existingTemplates[templateName] - - if (template == existingTemplate) return currentState - - // find templates with overlapping index pattern - val overlaps = findConflictingISMTemplates(templateName, template.indexPatterns, template.priority, existingTemplates) - if (overlaps.isNotEmpty()) { - val esg = "new ism template $templateName has index pattern ${template.indexPatterns} " + - "matching existing templates ${overlaps.entries.stream().map { "${it.key} => ${it.value}" }.collect(Collectors.joining(","))}," + - " please use a different priority than ${template.priority}" - throw IllegalArgumentException(esg) - } - - validateFormat(templateName, template.indexPatterns) - - return ClusterState.builder(currentState).metadata(Metadata.builder(currentState.metadata()) - .putISMTemplate(templateName, template, existingTemplates)).build() - } - - /** - * remove ISM template from cluster state metadata - */ - fun deleteISMTemplate(templateName: String, masterTimeout: TimeValue, listener: ActionListener<AcknowledgedResponse>) { - clusterService.submitStateUpdateTask( - IndexManagementPlugin.PLUGIN_NAME, - object : ClusterStateUpdateTask(Priority.NORMAL) { - override fun execute(currentState: ClusterState): ClusterState { - val existingTemplates = currentState.metadata.ismTemplates() - return ClusterState.builder(currentState).metadata(Metadata.builder(currentState.metadata) - .removeISMTemplate(templateName, existingTemplates)).build() - } - - override fun onFailure(source: String, e: Exception) { - listener.onFailure(e) - } - - override fun timeout(): TimeValue = masterTimeout - - override fun clusterStateProcessed(source: String, oldState: ClusterState, newState: ClusterState) { - listener.onResponse(AcknowledgedResponse(true)) - } - } - ) - } - +class ISMTemplateService { companion object { /** * find the matching template for the index @@ -136,7 +38,7 @@ class ISMTemplateService @Inject constructor( * * @param ismTemplates current ISM templates saved in metadata * @param indexMetadata cluster state index metadata - * @return template name matching with given index + * @return policyID */ @Suppress("ReturnCount") fun findMatchingISMTemplate(ismTemplates: Map<String, ISMTemplate>, indexMetadata: IndexMetadata): String? { @@ -172,26 +74,8 @@ class ISMTemplateService @Inject constructor( * reusing ES validate function in MetadataIndexTemplateService */ @Suppress("ComplexMethod") - fun validateFormat(templateName: String, indexPatterns: List<String>) { + fun validateFormat(indexPatterns: List<String>): ElasticsearchException? { val validationErrors = mutableListOf<String>() - if (templateName.contains(" ")) { - validationErrors.add("name must not contain a space") - } - if (templateName.contains(",")) { - validationErrors.add("name must not contain a ','") - } - if (templateName.contains("#")) { - validationErrors.add("name must not contain a '#'") - } - if (templateName.contains("*")) { - validationErrors.add("name must not contain a '*'") - } - if (templateName.startsWith("_")) { - validationErrors.add("name must not start with '_'") - } - if (templateName.toLowerCase(Locale.ROOT) != templateName) { - validationErrors.add("name must be lower cased") - } for (indexPattern in indexPatterns) { if (indexPattern.contains(" ")) { validationErrors.add("index_patterns [$indexPattern] must not contain a space") @@ -217,8 +101,9 @@ class ISMTemplateService @Inject constructor( if (validationErrors.size > 0) { val validationException = ValidationException() validationException.addValidationErrors(validationErrors) - throw InvalidIndexTemplateException(templateName, validationException.message) + return IndexManagementException.wrap(validationException) } + return null } /** @@ -231,17 +116,18 @@ class ISMTemplateService @Inject constructor( candidate: String, indexPatterns: List<String>, priority: Int, - ismTemplates: Map<String, ISMTemplate> + ismTemplates: Map<String, ISMTemplate?> ): Map<String, List<String>> { val automaton1 = Regex.simpleMatchToAutomaton(*indexPatterns.toTypedArray()) val overlappingTemplates = mutableMapOf<String, List<String>>() // focus on template with same priority - ismTemplates.filter { it.value.priority == priority }.forEach { (templateName, template) -> + ismTemplates.filterNotNullValues() + .filter { it.value.priority == priority }.forEach { (policyID, template) -> val automaton2 = Regex.simpleMatchToAutomaton(*template.indexPatterns.toTypedArray()) if (!Operations.isEmpty(Operations.intersection(automaton1, automaton2))) { - log.info("existing template $templateName overlaps candidate $candidate") - overlappingTemplates[templateName] = template.indexPatterns + log.info("existing ism_template in $policyID overlaps candidate $candidate") + overlappingTemplates[policyID] = template.indexPatterns } } overlappingTemplates.remove(candidate) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt index ba6a477dd..bd621050b 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt @@ -17,6 +17,7 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanageme import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementIndices +import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.parseWithType import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ISMTemplateService.Companion.findMatchingISMTemplate import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.getClusterStateManagedIndexConfig @@ -24,9 +25,12 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagemen import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.getPolicyID import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.retry import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.suspendUntil +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.filterNotNullValues +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.ismTemplatesFromSearchResponse import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.shouldCreateManagedIndexConfig import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.shouldDeleteManagedIndexConfig import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.shouldDeleteManagedIndexMetaData +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.coordinator.ClusterStateManagedIndexConfig import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData @@ -39,6 +43,7 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagemen import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings.Companion.SWEEP_PERIOD import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.updateindexmetadata.UpdateManagedIndexMetaDataAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.updateindexmetadata.UpdateManagedIndexMetaDataRequest +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.ISM_TEMPLATE_FIELD import com.amazon.opendistroforelasticsearch.indexmanagement.util.OpenForTesting import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.managedIndexConfigIndexRequest import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.deleteManagedIndexRequest @@ -47,7 +52,6 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagemen import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.getSweptManagedIndexSearchRequest import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.isFailed import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.isPolicyCompleted -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.ismTemplates import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.updateEnableManagedIndexRequest import com.amazon.opendistroforelasticsearch.indexmanagement.util.NO_ID import kotlinx.coroutines.CoroutineName @@ -63,6 +67,7 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse import org.elasticsearch.action.bulk.BackoffPolicy import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.search.SearchRequest import org.elasticsearch.action.search.SearchResponse import org.elasticsearch.action.support.IndicesOptions import org.elasticsearch.action.support.master.AcknowledgedResponse @@ -70,6 +75,7 @@ import org.elasticsearch.client.Client import org.elasticsearch.cluster.ClusterChangedEvent import org.elasticsearch.cluster.ClusterState import org.elasticsearch.cluster.ClusterStateListener +import org.elasticsearch.cluster.block.ClusterBlockException import org.elasticsearch.cluster.service.ClusterService import org.elasticsearch.common.bytes.BytesReference import org.elasticsearch.common.component.LifecycleListener @@ -81,7 +87,10 @@ import org.elasticsearch.common.xcontent.XContentHelper import org.elasticsearch.common.xcontent.XContentParser import org.elasticsearch.common.xcontent.XContentType import org.elasticsearch.index.Index +import org.elasticsearch.index.IndexNotFoundException +import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.rest.RestStatus +import org.elasticsearch.search.builder.SearchSourceBuilder import org.elasticsearch.threadpool.Scheduler import org.elasticsearch.threadpool.ThreadPool @@ -291,26 +300,45 @@ class ManagedIndexCoordinator( /** * build requests to create jobs for indices matching ISM templates */ - fun getMatchingIndicesUpdateReqs(clusterState: ClusterState, indexNames: List<String>): List<DocWriteRequest<*>> { + suspend fun getMatchingIndicesUpdateReqs(clusterState: ClusterState, indexNames: List<String>): List<DocWriteRequest<*>> { val indexMetadatas = clusterState.metadata.indices - val templates = clusterState.metadata.ismTemplates() + val templates = getISMTemplates() - val indexToMatchMap = indexNames.map { indexName -> + val indexToMatchedPolicy = indexNames.map { indexName -> indexName to findMatchingISMTemplate(templates, indexMetadatas[indexName]) }.toMap() val updateManagedIndexReqs = mutableListOf<DocWriteRequest<*>>() - indexToMatchMap.filter { (_, template) -> template != null }.forEach { (index, template) -> + indexToMatchedPolicy.filterNotNullValues() + .forEach { (index, policyID) -> val indexUuid = indexMetadatas[index].indexUUID - val policyID = templates[template]?.policyID - if (indexUuid != null && policyID != null) { - updateManagedIndexReqs.add(managedIndexConfigIndexRequest(index, indexUuid, policyID, jobInterval)) + if (indexUuid != null) { + logger.info("auto manage index $index to policy $policyID") + updateManagedIndexReqs.add( + managedIndexConfigIndexRequest(index, indexUuid, policyID, jobInterval)) } } return updateManagedIndexReqs } + suspend fun getISMTemplates(): Map<String, ISMTemplate> { + val searchRequest = SearchRequest() + .source( + SearchSourceBuilder().query( + QueryBuilders.existsQuery(ISM_TEMPLATE_FIELD))) + .indices(INDEX_MANAGEMENT_INDEX) + + return try { + val response: SearchResponse = client.suspendUntil { search(searchRequest, it) } + ismTemplatesFromSearchResponse(response).filterNotNullValues() + } catch (ex: IndexNotFoundException) { + emptyMap() + } catch (ex: ClusterBlockException) { + emptyMap() + } + } + /** * Background sweep process that periodically sweeps for updates to ManagedIndices * diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/elasticapi/ElasticExtensions.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/elasticapi/ElasticExtensions.kt index ab2d32b28..e7bc7ca69 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/elasticapi/ElasticExtensions.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/elasticapi/ElasticExtensions.kt @@ -17,10 +17,18 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi +import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.parseWithType +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.Policy import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.coordinator.ClusterStateManagedIndexConfig import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings +import org.elasticsearch.action.search.SearchResponse import org.elasticsearch.cluster.metadata.IndexMetadata +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler +import org.elasticsearch.common.xcontent.NamedXContentRegistry +import org.elasticsearch.common.xcontent.XContentFactory +import org.elasticsearch.common.xcontent.XContentType /** * Compares current and previous IndexMetaData to determine if we should create [ManagedIndexConfig]. @@ -98,3 +106,26 @@ fun IndexMetadata.getManagedIndexMetaData(): ManagedIndexMetaData? { } return null } + +/** + * Do a exists search query to retrieve all policy with ism_template field + * parse search response with this function + * + * @return map of policyID to ISMTemplate in this policy + */ +fun ismTemplatesFromSearchResponse(response: SearchResponse, xContentRegistry: NamedXContentRegistry = NamedXContentRegistry.EMPTY): + Map<String, ISMTemplate?> { + return response.hits.hits.map { + val id = it.id + val seqNo = it.seqNo + val primaryTerm = it.primaryTerm + val xcp = XContentFactory.xContent(XContentType.JSON) + .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, it.sourceAsString) + xcp.parseWithType(id, seqNo, primaryTerm, Policy.Companion::parse) + .copy(id = id, seqNo = seqNo, primaryTerm = primaryTerm) + }.map { it.id to it.ismTemplate }.toMap() +} + +@Suppress("UNCHECKED_CAST") +fun <K, V> Map<K, V?>.filterNotNullValues(): Map<K, V> = + filterValues { it != null } as Map<K, V> diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplate.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplate.kt index dcbed958d..0173459b7 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplate.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplate.kt @@ -18,10 +18,9 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanageme import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.instant import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.optionalTimeField import org.apache.logging.log4j.LogManager -import org.elasticsearch.cluster.AbstractDiffable -import org.elasticsearch.cluster.Diff import org.elasticsearch.common.io.stream.StreamInput import org.elasticsearch.common.io.stream.StreamOutput +import org.elasticsearch.common.io.stream.Writeable import org.elasticsearch.common.xcontent.ToXContent import org.elasticsearch.common.xcontent.ToXContentObject import org.elasticsearch.common.xcontent.XContentBuilder @@ -34,14 +33,11 @@ import java.time.Instant private val log = LogManager.getLogger(ISMTemplate::class.java) -// ComposableIndexTemplate -// ManagedIndexMetaData data class ISMTemplate( val indexPatterns: List<String>, - val policyID: String, val priority: Int, val lastUpdatedTime: Instant -) : ToXContentObject, AbstractDiffable<ISMTemplate>() { +) : ToXContentObject, Writeable { init { require(indexPatterns.isNotEmpty()) { "at least give one index pattern" } @@ -50,7 +46,6 @@ data class ISMTemplate( override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { return builder.startObject() .field(INDEX_PATTERN, indexPatterns) - .field(POLICY_ID, policyID) .field(PRIORITY, priority) .optionalTimeField(LAST_UPDATED_TIME_FIELD, lastUpdatedTime) .endObject() @@ -59,7 +54,6 @@ data class ISMTemplate( @Throws(IOException::class) constructor(sin: StreamInput) : this( sin.readStringList(), - sin.readString(), sin.readInt(), sin.readInstant() ) @@ -67,27 +61,23 @@ data class ISMTemplate( @Throws(IOException::class) override fun writeTo(out: StreamOutput) { out.writeStringCollection(indexPatterns) - out.writeString(policyID) out.writeInt(priority) out.writeInstant(lastUpdatedTime) } companion object { - const val ISM_TEMPLATE_ID = "template_name" const val ISM_TEMPLATE_TYPE = "ism_template" const val INDEX_PATTERN = "index_patterns" - const val POLICY_ID = "policy_id" const val PRIORITY = "priority" const val LAST_UPDATED_TIME_FIELD = "last_updated_time" @Suppress("ComplexMethod") fun parse(xcp: XContentParser): ISMTemplate { val indexPatterns: MutableList<String> = mutableListOf() - var policyID: String? = null var priority = 0 var lastUpdatedTime: Instant? = null - ensureExpectedToken(Token.START_OBJECT, xcp.nextToken(), xcp) + ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) while (xcp.nextToken() != Token.END_OBJECT) { val fieldName = xcp.currentName() xcp.nextToken() @@ -99,7 +89,6 @@ data class ISMTemplate( indexPatterns.add(xcp.text()) } } - POLICY_ID -> policyID = xcp.text() PRIORITY -> priority = if (xcp.currentToken() == Token.VALUE_NULL) 0 else xcp.intValue() LAST_UPDATED_TIME_FIELD -> lastUpdatedTime = xcp.instant() else -> throw IllegalArgumentException("Invalid field: [$fieldName] found in ISMTemplate.") @@ -107,13 +96,10 @@ data class ISMTemplate( } return ISMTemplate( - indexPatterns, - requireNotNull(policyID) { "policy id is null" }, - priority, - lastUpdatedTime ?: Instant.now() + indexPatterns = indexPatterns, + priority = priority, + lastUpdatedTime = lastUpdatedTime ?: Instant.now() ) } - - fun readISMTemplateDiffFrom(sin: StreamInput): Diff<ISMTemplate> = AbstractDiffable.readDiffFrom(::ISMTemplate, sin) } } diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateMetadata.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateMetadata.kt deleted file mode 100644 index bc39aae64..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateMetadata.kt +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model - -import org.apache.logging.log4j.LogManager -import org.elasticsearch.Version -import org.elasticsearch.cluster.Diff -import org.elasticsearch.cluster.DiffableUtils -import org.elasticsearch.cluster.NamedDiff -import org.elasticsearch.cluster.metadata.Metadata -import org.elasticsearch.common.ParseField -import org.elasticsearch.common.io.stream.StreamInput -import org.elasticsearch.common.io.stream.StreamOutput -import org.elasticsearch.common.xcontent.ToXContent -import org.elasticsearch.common.xcontent.XContentBuilder -import java.io.IOException -import java.util.EnumSet - -private val log = LogManager.getLogger(ISMTemplateMetadata::class.java) - -/** - * <template_name>: ISMTemplate - */ -// ComponentTemplateMetadata -// EnrichMetadata -class ISMTemplateMetadata(val ismTemplates: Map<String, ISMTemplate>) : Metadata.Custom { - - @Throws(IOException::class) - constructor(sin: StreamInput) : this( - sin.readMap(StreamInput::readString, ::ISMTemplate) - ) - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - out.writeMap(ismTemplates, StreamOutput::writeString) { stream, `val` -> `val`.writeTo(stream) } - } - - override fun diff(before: Metadata.Custom): Diff<Metadata.Custom> { - return ISMTemplateMetadataDiff((before as ISMTemplateMetadata), this) - } - - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - builder.startObject(ISM_TEMPLATE.preferredName) - ismTemplates.forEach { (k, v) -> - builder.field(k, v) - } - builder.endObject() - return builder - } - - override fun getWriteableName(): String = TYPE - - override fun getMinimalSupportedVersion(): Version = Version.V_7_7_0 - - override fun context(): EnumSet<Metadata.XContentContext> = Metadata.ALL_CONTEXTS - - class ISMTemplateMetadataDiff : NamedDiff<Metadata.Custom> { - - val ismTemplateDiff: Diff<Map<String, ISMTemplate>> - - constructor(before: ISMTemplateMetadata, after: ISMTemplateMetadata) { - this.ismTemplateDiff = DiffableUtils.diff(before.ismTemplates, after.ismTemplates, DiffableUtils.getStringKeySerializer()) - } - - @Throws(IOException::class) - constructor(sin: StreamInput) { - this.ismTemplateDiff = DiffableUtils.readJdkMapDiff(sin, DiffableUtils.getStringKeySerializer(), - ::ISMTemplate, ISMTemplate.Companion::readISMTemplateDiffFrom) - } - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - ismTemplateDiff.writeTo(out) - } - - override fun getWriteableName(): String = TYPE - - override fun apply(part: Metadata.Custom): Metadata.Custom { - return ISMTemplateMetadata(ismTemplateDiff.apply((part as ISMTemplateMetadata).ismTemplates)) - } - } - - companion object { - val TYPE = "ism_template" - val ISM_TEMPLATE = ParseField("ism_template") - - fun readDiffFrom(sin: StreamInput): NamedDiff<Metadata.Custom> = ISMTemplateMetadataDiff(sin) - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/Policy.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/Policy.kt index 69b2e79aa..e68261165 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/Policy.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/Policy.kt @@ -16,10 +16,10 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.instant +import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.optionalISMTemplateField import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.optionalTimeField import com.amazon.opendistroforelasticsearch.indexmanagement.util.IndexUtils import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.WITH_TYPE -import org.apache.logging.log4j.LogManager import org.elasticsearch.common.io.stream.StreamInput import org.elasticsearch.common.io.stream.StreamOutput import org.elasticsearch.common.io.stream.Writeable @@ -33,8 +33,6 @@ import org.elasticsearch.index.seqno.SequenceNumbers import java.io.IOException import java.time.Instant -private val log = LogManager.getLogger(Policy::class.java) - data class Policy( val id: String = NO_ID, val seqNo: Long = SequenceNumbers.UNASSIGNED_SEQ_NO, @@ -44,7 +42,8 @@ data class Policy( val lastUpdatedTime: Instant, val errorNotification: ErrorNotification?, val defaultState: String, - val states: List<State> + val states: List<State>, + val ismTemplate: ISMTemplate? = null ) : ToXContentObject, Writeable { init { @@ -75,6 +74,7 @@ data class Policy( .field(ERROR_NOTIFICATION_FIELD, errorNotification) .field(DEFAULT_STATE_FIELD, defaultState) .field(STATES_FIELD, states.toTypedArray()) + .optionalISMTemplateField(ISM_TEMPLATE, ismTemplate) if (params.paramAsBoolean(WITH_TYPE, true)) builder.endObject() return builder.endObject() } @@ -89,7 +89,8 @@ data class Policy( lastUpdatedTime = sin.readInstant(), errorNotification = sin.readOptionalWriteable(::ErrorNotification), defaultState = sin.readString(), - states = sin.readList(::State) + states = sin.readList(::State), + ismTemplate = sin.readOptionalWriteable(::ISMTemplate) ) @Throws(IOException::class) @@ -103,6 +104,7 @@ data class Policy( out.writeOptionalWriteable(errorNotification) out.writeString(defaultState) out.writeList(states) + out.writeOptionalWriteable(ismTemplate) } companion object { @@ -115,6 +117,7 @@ data class Policy( const val ERROR_NOTIFICATION_FIELD = "error_notification" const val DEFAULT_STATE_FIELD = "default_state" const val STATES_FIELD = "states" + const val ISM_TEMPLATE = "ism_template" @Suppress("ComplexMethod") @JvmStatic @@ -132,6 +135,7 @@ data class Policy( var lastUpdatedTime: Instant? = null var schemaVersion: Long = IndexUtils.DEFAULT_SCHEMA_VERSION val states: MutableList<State> = mutableListOf() + var ismTemplate: ISMTemplate? = null ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) while (xcp.nextToken() != Token.END_OBJECT) { @@ -151,6 +155,7 @@ data class Policy( states.add(State.parse(xcp)) } } + ISM_TEMPLATE -> ismTemplate = if (xcp.currentToken() == Token.VALUE_NULL) null else ISMTemplate.parse(xcp) else -> throw IllegalArgumentException("Invalid field: [$fieldName] found in Policy.") } } @@ -164,7 +169,8 @@ data class Policy( lastUpdatedTime ?: Instant.now(), errorNotification, requireNotNull(defaultState) { "$DEFAULT_STATE_FIELD is null" }, - states.toList() + states.toList(), + ismTemplate ) } } diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestAddISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestAddISMTemplateAction.kt deleted file mode 100644 index fe3774634..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestAddISMTemplateAction.kt +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler - -import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin.Companion.ISM_TEMPLATE_BASE_URI -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put.PutISMTemplateAction -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put.PutISMTemplateRequest -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put.PutISMTemplateResponse -import org.apache.logging.log4j.LogManager -import org.elasticsearch.action.support.master.MasterNodeRequest.DEFAULT_MASTER_NODE_TIMEOUT -import org.elasticsearch.client.node.NodeClient -import org.elasticsearch.common.xcontent.ToXContent -import org.elasticsearch.rest.BaseRestHandler -import org.elasticsearch.rest.BytesRestResponse -import org.elasticsearch.rest.RestHandler.Route -import org.elasticsearch.rest.RestRequest -import org.elasticsearch.rest.RestRequest.Method.PUT -import org.elasticsearch.rest.RestResponse -import org.elasticsearch.rest.RestStatus -import org.elasticsearch.rest.action.RestResponseListener -import java.lang.IllegalArgumentException -import java.time.Instant - -private val log = LogManager.getLogger(RestAddISMTemplateAction::class.java) - -class RestAddISMTemplateAction : BaseRestHandler() { - override fun routes(): List<Route> { - return listOf( - Route(PUT, ISM_TEMPLATE_BASE_URI), - Route(PUT, "$ISM_TEMPLATE_BASE_URI/{templateID}") - ) - } - - override fun getName(): String = "add_ism_template_action" - - override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { - log.info("${request.method()} $ISM_TEMPLATE_BASE_URI") - - val templateName = request.param("templateID", "") - if (templateName == "") { throw IllegalArgumentException("Missing template name") } - - val xcp = request.contentParser() - val ismTemplate = ISMTemplate.parse(xcp).copy(lastUpdatedTime = Instant.now()) - val masterTimeout = request.paramAsTime("master_timeout", DEFAULT_MASTER_NODE_TIMEOUT) - val addISMTemplateRequest = PutISMTemplateRequest(templateName, ismTemplate).masterNodeTimeout(masterTimeout) - - return RestChannelConsumer { channel -> - client.execute(PutISMTemplateAction.INSTANCE, addISMTemplateRequest, object : RestResponseListener<PutISMTemplateResponse>(channel) { - override fun buildResponse(response: PutISMTemplateResponse): RestResponse { - val restResponse = BytesRestResponse(response.status, response.toXContent(channel.newBuilder(), ToXContent.EMPTY_PARAMS)) - if (response.status == RestStatus.CREATED) { - val location = "$ISM_TEMPLATE_BASE_URI/${response.id}" - restResponse.addHeader("Location", location) - } - return restResponse - } - }) - } - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestDeleteISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestDeleteISMTemplateAction.kt deleted file mode 100644 index 74c709fe8..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestDeleteISMTemplateAction.kt +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler - -import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin.Companion.ISM_TEMPLATE_BASE_URI -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.delete.DeleteISMTemplateAction -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.delete.DeleteISMTemplateRequest -import org.apache.logging.log4j.LogManager -import org.elasticsearch.client.node.NodeClient -import org.elasticsearch.rest.BaseRestHandler -import org.elasticsearch.rest.RestHandler.Route -import org.elasticsearch.rest.RestRequest -import org.elasticsearch.rest.RestRequest.Method.DELETE -import org.elasticsearch.rest.action.RestToXContentListener -import java.lang.IllegalArgumentException - -private val log = LogManager.getLogger(RestDeleteISMTemplateAction::class.java) - -class RestDeleteISMTemplateAction : BaseRestHandler() { - override fun routes(): List<Route> { - return listOf( - Route(DELETE, "$ISM_TEMPLATE_BASE_URI/{templateID}") - ) - } - - override fun getName(): String = "remove_ism_template_action" - - override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { - log.info("${request.method()} $ISM_TEMPLATE_BASE_URI") - - val templateName = request.param("templateID", "") - if (templateName == "") { throw IllegalArgumentException("Missing template name") } - - val deleteISMTemplateRequest = DeleteISMTemplateRequest(templateName) - - return RestChannelConsumer { channel -> - client.execute(DeleteISMTemplateAction.INSTANCE, deleteISMTemplateRequest, RestToXContentListener(channel)) - } - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestGetISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestGetISMTemplateAction.kt deleted file mode 100644 index 0c8c2a05c..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestGetISMTemplateAction.kt +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler - -import org.apache.logging.log4j.LogManager -import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin.Companion.ISM_TEMPLATE_BASE_URI -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get.GetISMTemplateAction -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get.GetISMTemplateRequest -import org.elasticsearch.action.support.master.MasterNodeRequest.DEFAULT_MASTER_NODE_TIMEOUT -import org.elasticsearch.client.node.NodeClient -import org.elasticsearch.common.Strings -import org.elasticsearch.rest.BaseRestHandler -import org.elasticsearch.rest.RestHandler.Route -import org.elasticsearch.rest.RestRequest -import org.elasticsearch.rest.RestRequest.Method.GET -import org.elasticsearch.rest.action.RestToXContentListener - -private val log = LogManager.getLogger(RestGetISMTemplateAction::class.java) - -class RestGetISMTemplateAction : BaseRestHandler() { - override fun routes(): List<Route> { - return listOf( - Route(GET, ISM_TEMPLATE_BASE_URI), - Route(GET, "$ISM_TEMPLATE_BASE_URI/{templateID}") - ) - } - - override fun getName(): String = "get_ism_template_action" - - override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { - log.info("${request.method()} $ISM_TEMPLATE_BASE_URI") - - val templateNames = Strings.splitStringByCommaToArray(request.param("templateID")) - val masterTimeout = request.paramAsTime("master_timeout", DEFAULT_MASTER_NODE_TIMEOUT) - val getISMTemplateReq = GetISMTemplateRequest(templateNames).masterNodeTimeout(masterTimeout) - - return RestChannelConsumer { channel -> - client.execute(GetISMTemplateAction.INSTANCE, getISMTemplateReq, RestToXContentListener(channel)) - } - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/TransportIndexPolicyAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/TransportIndexPolicyAction.kt index 4a7f62e97..fc10a6ba6 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/TransportIndexPolicyAction.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/TransportIndexPolicyAction.kt @@ -17,6 +17,11 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanageme import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementIndices import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ISMTemplateService.Companion.findConflictingISMTemplates +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ISMTemplateService.Companion.validateFormat +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.ismTemplatesFromSearchResponse +import com.amazon.opendistroforelasticsearch.indexmanagement.util.IndexManagementException +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.ISM_TEMPLATE_FIELD import com.amazon.opendistroforelasticsearch.indexmanagement.util.IndexUtils import org.apache.logging.log4j.LogManager import org.elasticsearch.ElasticsearchStatusException @@ -24,16 +29,22 @@ import org.elasticsearch.action.ActionListener import org.elasticsearch.action.DocWriteRequest import org.elasticsearch.action.index.IndexRequest import org.elasticsearch.action.index.IndexResponse +import org.elasticsearch.action.search.SearchRequest +import org.elasticsearch.action.search.SearchResponse import org.elasticsearch.action.support.ActionFilters import org.elasticsearch.action.support.HandledTransportAction import org.elasticsearch.action.support.master.AcknowledgedResponse import org.elasticsearch.client.node.NodeClient import org.elasticsearch.common.inject.Inject +import org.elasticsearch.common.xcontent.NamedXContentRegistry import org.elasticsearch.common.xcontent.XContentFactory +import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.index.seqno.SequenceNumbers import org.elasticsearch.rest.RestStatus +import org.elasticsearch.search.builder.SearchSourceBuilder import org.elasticsearch.tasks.Task import org.elasticsearch.transport.TransportService +import java.util.stream.Collectors private val log = LogManager.getLogger(TransportIndexPolicyAction::class.java) @@ -41,7 +52,8 @@ class TransportIndexPolicyAction @Inject constructor( val client: NodeClient, transportService: TransportService, actionFilters: ActionFilters, - val ismIndices: IndexManagementIndices + val ismIndices: IndexManagementIndices, + val xContentRegistry: NamedXContentRegistry ) : HandledTransportAction<IndexPolicyRequest, IndexPolicyResponse>( IndexPolicyAction.NAME, transportService, actionFilters, ::IndexPolicyRequest ) { @@ -69,7 +81,12 @@ class TransportIndexPolicyAction @Inject constructor( private fun onCreateMappingsResponse(response: AcknowledgedResponse) { if (response.isAcknowledged) { log.info("Successfully created or updated ${IndexManagementPlugin.INDEX_MANAGEMENT_INDEX} with newest mappings.") - putPolicy() + + // if there is template field, we will check + val reqTemplate = request.policy.ismTemplate + if (reqTemplate != null) { + checkTemplate(reqTemplate.indexPatterns, reqTemplate.priority) + } else putPolicy() } else { log.error("Unable to create or update ${IndexManagementPlugin.INDEX_MANAGEMENT_INDEX} with newest mapping.") @@ -79,6 +96,41 @@ class TransportIndexPolicyAction @Inject constructor( } } + private fun checkTemplate(indexPatterns: List<String>, priority: Int) { + val possibleEx = validateFormat(indexPatterns) + if (possibleEx != null) { + actionListener.onFailure(possibleEx) + return + } + + val searchRequest = SearchRequest() + .source( + SearchSourceBuilder().query( + QueryBuilders.existsQuery(ISM_TEMPLATE_FIELD))) + .indices(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX) + + client.search(searchRequest, object : ActionListener<SearchResponse> { + override fun onResponse(response: SearchResponse) { + val ismTemplates = ismTemplatesFromSearchResponse(response, xContentRegistry) + val overlaps = findConflictingISMTemplates(request.policyID, indexPatterns, priority, ismTemplates) + if (overlaps.isNotEmpty()) { + val esg = "new policy ${request.policyID} has an ism template with index pattern $indexPatterns " + + "matching existing templates ${overlaps.entries.stream().map { "${it.key} => ${it.value}" }.collect( + Collectors.joining(","))}," + + " please use a different priority than $priority" + actionListener.onFailure(IndexManagementException.wrap(IllegalArgumentException(esg))) + return + } + + putPolicy() + } + + override fun onFailure(t: Exception) { + actionListener.onFailure(t) + } + }) + } + private fun putPolicy() { request.policy.copy(schemaVersion = IndexUtils.indexManagementConfigSchemaVersion) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateAction.kt deleted file mode 100644 index b90840132..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateAction.kt +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.delete - -import org.elasticsearch.action.ActionType -import org.elasticsearch.action.support.master.AcknowledgedResponse - -class DeleteISMTemplateAction : ActionType<AcknowledgedResponse>(NAME, ::AcknowledgedResponse) { - companion object { - val NAME = "cluster:admin/opendistro/ism/templates/remove" - val INSTANCE = DeleteISMTemplateAction() - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequest.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequest.kt deleted file mode 100644 index 4a767f862..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequest.kt +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.delete - -import org.elasticsearch.action.ActionRequestValidationException -import org.elasticsearch.action.support.master.MasterNodeRequest -import org.elasticsearch.common.io.stream.StreamInput -import org.elasticsearch.common.io.stream.StreamOutput -import java.io.IOException - -class DeleteISMTemplateRequest : MasterNodeRequest<DeleteISMTemplateRequest> { - - val templateName: String - - constructor( - templateName: String - ) : super() { - this.templateName = templateName - } - - @Throws(IOException::class) - constructor(sin: StreamInput) : super(sin) { - templateName = sin.readString() - } - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - super.writeTo(out) - out.writeString(templateName) - } - - override fun validate(): ActionRequestValidationException? { - return null - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/TransportDeleteISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/TransportDeleteISMTemplateAction.kt deleted file mode 100644 index 3fa4f2b02..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/TransportDeleteISMTemplateAction.kt +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.delete - -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ISMTemplateService -import org.apache.logging.log4j.LogManager -import org.elasticsearch.action.ActionListener -import org.elasticsearch.action.support.ActionFilters -import org.elasticsearch.action.support.master.AcknowledgedResponse -import org.elasticsearch.action.support.master.TransportMasterNodeAction -import org.elasticsearch.cluster.ClusterState -import org.elasticsearch.cluster.block.ClusterBlockException -import org.elasticsearch.cluster.block.ClusterBlockLevel -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver -import org.elasticsearch.cluster.service.ClusterService -import org.elasticsearch.common.inject.Inject -import org.elasticsearch.common.io.stream.StreamInput -import org.elasticsearch.common.io.stream.Writeable -import org.elasticsearch.threadpool.ThreadPool -import org.elasticsearch.transport.TransportService - -private val log = LogManager.getLogger(TransportDeleteISMTemplateAction::class.java) - -class TransportDeleteISMTemplateAction @Inject constructor( - transportService: TransportService, - clusterService: ClusterService, - threadPool: ThreadPool, - actionFilters: ActionFilters, - indexNameExpressionResolver: IndexNameExpressionResolver, - val ismTemplateService: ISMTemplateService -) : TransportMasterNodeAction<DeleteISMTemplateRequest, AcknowledgedResponse>( - DeleteISMTemplateAction.NAME, - transportService, - clusterService, - threadPool, - actionFilters, - Writeable.Reader { DeleteISMTemplateRequest(it) }, - indexNameExpressionResolver -) { - override fun executor(): String { - return ThreadPool.Names.SAME - } - - override fun read(sin: StreamInput): AcknowledgedResponse { - return AcknowledgedResponse(sin) - } - - override fun masterOperation(request: DeleteISMTemplateRequest, state: ClusterState, listener: ActionListener<AcknowledgedResponse>) { - ismTemplateService.deleteISMTemplate(request.templateName, request.masterNodeTimeout(), listener) - } - - override fun checkBlock(request: DeleteISMTemplateRequest, state: ClusterState): ClusterBlockException? { - return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE) - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateAction.kt deleted file mode 100644 index ebf148caa..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateAction.kt +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get - -import org.elasticsearch.action.ActionType - -class GetISMTemplateAction : ActionType<GetISMTemplateResponse>(NAME, ::GetISMTemplateResponse) { - companion object { - val NAME = "cluster:admin/opendistro/ism/templates/read" - val INSTANCE = GetISMTemplateAction() - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequest.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequest.kt deleted file mode 100644 index c3e7895ce..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequest.kt +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get - -import org.elasticsearch.action.ActionRequestValidationException -import org.elasticsearch.action.support.master.MasterNodeRequest -import org.elasticsearch.common.io.stream.StreamInput -import org.elasticsearch.common.io.stream.StreamOutput -import java.io.IOException - -class GetISMTemplateRequest : MasterNodeRequest<GetISMTemplateRequest> { - - // TODO not sure array is the right choice - val templateNames: Array<String> - - constructor(templateName: Array<String>) : super() { - this.templateNames = templateName - } - - @Throws(IOException::class) - constructor(sin: StreamInput) : super(sin) { - templateNames = sin.readStringArray() - } - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - super.writeTo(out) - out.writeStringArray(templateNames) - } - - override fun validate(): ActionRequestValidationException? { - return null - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponse.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponse.kt deleted file mode 100644 index 628e696b1..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponse.kt +++ /dev/null @@ -1,52 +0,0 @@ -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get - -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate -import org.elasticsearch.action.ActionResponse -import org.elasticsearch.common.io.stream.StreamInput -import org.elasticsearch.common.io.stream.StreamOutput -import org.elasticsearch.common.xcontent.ToXContent -import org.elasticsearch.common.xcontent.ToXContentObject -import org.elasticsearch.common.xcontent.XContentBuilder -import java.io.IOException - -// GetComposableIndexTemplateAction.Response -class GetISMTemplateResponse : ActionResponse, ToXContentObject { - - val ismTemplates: Map<String, ISMTemplate> - - constructor(ismTemplates: Map<String, ISMTemplate>) : super() { - this.ismTemplates = ismTemplates - } - - @Throws(IOException::class) - constructor(sin: StreamInput) : super(sin) { - val size = sin.readVInt() - ismTemplates = mutableMapOf() - repeat(size) { - ismTemplates.put(sin.readString(), ISMTemplate(sin)) - } - } - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - out.writeVInt(ismTemplates.size) - ismTemplates.forEach { (k, v) -> - out.writeString(k) - v.writeTo(out) - } - } - - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - builder.startObject().startArray(ISM_TEMPLATES) - ismTemplates.forEach { (k, v) -> - builder.startObject().field(TEMPLATE_NAME, k).field(ISM_TEMPLATE, v).endObject() - } - return builder.endArray().endObject() - } - - companion object { - val ISM_TEMPLATES = "ism_templates" - val TEMPLATE_NAME = "template_name" - val ISM_TEMPLATE = "ism_template" - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/TransportGetISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/TransportGetISMTemplateAction.kt deleted file mode 100644 index 7a811aefc..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/TransportGetISMTemplateAction.kt +++ /dev/null @@ -1,74 +0,0 @@ -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get - -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.ismTemplates -import org.apache.logging.log4j.LogManager -import org.elasticsearch.ResourceNotFoundException -import org.elasticsearch.action.ActionListener -import org.elasticsearch.action.support.ActionFilters -import org.elasticsearch.action.support.master.TransportMasterNodeAction -import org.elasticsearch.cluster.ClusterState -import org.elasticsearch.cluster.block.ClusterBlockException -import org.elasticsearch.cluster.block.ClusterBlockLevel -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver -import org.elasticsearch.cluster.service.ClusterService -import org.elasticsearch.common.inject.Inject -import org.elasticsearch.common.io.stream.StreamInput -import org.elasticsearch.common.io.stream.Writeable -import org.elasticsearch.common.regex.Regex -import org.elasticsearch.threadpool.ThreadPool -import org.elasticsearch.transport.TransportService - -private val log = LogManager.getLogger(TransportGetISMTemplateAction::class.java) - -// TransportGetComposableIndexTemplateAction -class TransportGetISMTemplateAction @Inject constructor( - transportService: TransportService, - clusterService: ClusterService, - threadPool: ThreadPool, - actionFilters: ActionFilters, - indexNameExpressionResolver: IndexNameExpressionResolver -) : TransportMasterNodeAction<GetISMTemplateRequest, GetISMTemplateResponse>( - GetISMTemplateAction.NAME, - transportService, - clusterService, - threadPool, - actionFilters, - Writeable.Reader { GetISMTemplateRequest(it) }, - indexNameExpressionResolver -) { - override fun executor(): String { - return ThreadPool.Names.SAME - } - - override fun read(sin: StreamInput): GetISMTemplateResponse { - return GetISMTemplateResponse(sin) - } - - override fun masterOperation(request: GetISMTemplateRequest, state: ClusterState, listener: ActionListener<GetISMTemplateResponse>) { - val allTemplates = state.metadata.ismTemplates() - if (request.templateNames.isEmpty()) { - listener.onResponse(GetISMTemplateResponse(allTemplates)) - return - } - - val results = mutableMapOf<String, ISMTemplate>() - val reqTemplates = request.templateNames - if (allTemplates.isEmpty()) throw ResourceNotFoundException("index template matching ${reqTemplates.toList()} not found") - reqTemplates.forEach { name -> - allTemplates.forEach { (templateName, template) -> - when { - Regex.simpleMatch(name, templateName) -> results[templateName] = template - allTemplates.containsKey(name) -> results[templateName] = template - else -> throw ResourceNotFoundException("index template matching [$name] not found") - } - } - } - - listener.onResponse(GetISMTemplateResponse(results)) - } - - override fun checkBlock(request: GetISMTemplateRequest, state: ClusterState): ClusterBlockException? { - return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE) - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateAction.kt deleted file mode 100644 index af65d2c14..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateAction.kt +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put - -import org.elasticsearch.action.ActionType - -class PutISMTemplateAction : ActionType<PutISMTemplateResponse>(NAME, ::PutISMTemplateResponse) { - companion object { - val NAME = "cluster:admin/opendistro/ism/templates/add" - val INSTANCE = PutISMTemplateAction() - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateRequest.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateRequest.kt deleted file mode 100644 index 64b6ea0ae..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateRequest.kt +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put - -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate -import org.elasticsearch.action.ActionRequestValidationException -import org.elasticsearch.action.support.master.MasterNodeRequest -import org.elasticsearch.common.io.stream.StreamInput -import org.elasticsearch.common.io.stream.StreamOutput -import java.io.IOException - -class PutISMTemplateRequest : MasterNodeRequest<PutISMTemplateRequest> { - - val templateName: String - val ismTemplate: ISMTemplate - - constructor( - templateName: String, - ismTemplate: ISMTemplate - ) : super() { - this.templateName = templateName - this.ismTemplate = ismTemplate - } - - @Throws(IOException::class) - constructor(sin: StreamInput) : super(sin) { - templateName = sin.readString() - ismTemplate = ISMTemplate(sin) - } - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - super.writeTo(out) - out.writeString(templateName) - ismTemplate.writeTo(out) - } - - override fun validate(): ActionRequestValidationException? { - return null - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponse.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponse.kt deleted file mode 100644 index e3fd71411..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponse.kt +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put - -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate.Companion.ISM_TEMPLATE_ID -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate.Companion.ISM_TEMPLATE_TYPE -import org.elasticsearch.action.ActionResponse -import org.elasticsearch.common.io.stream.StreamInput -import org.elasticsearch.common.io.stream.StreamOutput -import org.elasticsearch.common.xcontent.ToXContent -import org.elasticsearch.common.xcontent.ToXContentObject -import org.elasticsearch.common.xcontent.XContentBuilder -import org.elasticsearch.rest.RestStatus -import java.io.IOException - -class PutISMTemplateResponse : ActionResponse, ToXContentObject { - - val id: String - val template: ISMTemplate - val status: RestStatus - - constructor( - id: String, - template: ISMTemplate, - status: RestStatus - ) : super() { - this.id = id - this.template = template - this.status = status - } - - @Throws(IOException::class) - constructor(sin: StreamInput) : this( - id = sin.readString(), - template = ISMTemplate(sin), - status = sin.readEnum(RestStatus::class.java) - ) - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - out.writeString(id) - template.writeTo(out) - out.writeEnum(status) - } - - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - return builder.startObject() - .field(ISM_TEMPLATE_ID, id) - .field(ISM_TEMPLATE_TYPE, template) - .endObject() - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/TransportPutISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/TransportPutISMTemplateAction.kt deleted file mode 100644 index 066328fd1..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/TransportPutISMTemplateAction.kt +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put - -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ISMTemplateService -import org.apache.logging.log4j.LogManager -import org.elasticsearch.action.ActionListener -import org.elasticsearch.action.support.ActionFilters -import org.elasticsearch.action.support.master.TransportMasterNodeAction -import org.elasticsearch.cluster.ClusterState -import org.elasticsearch.cluster.block.ClusterBlockException -import org.elasticsearch.cluster.block.ClusterBlockLevel -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver -import org.elasticsearch.cluster.service.ClusterService -import org.elasticsearch.common.inject.Inject -import org.elasticsearch.common.io.stream.StreamInput -import org.elasticsearch.common.io.stream.Writeable -import org.elasticsearch.threadpool.ThreadPool -import org.elasticsearch.transport.TransportService - -private val log = LogManager.getLogger(TransportPutISMTemplateAction::class.java) - -class TransportPutISMTemplateAction @Inject constructor( - transportService: TransportService, - clusterService: ClusterService, - threadPool: ThreadPool, - actionFilters: ActionFilters, - indexNameExpressionResolver: IndexNameExpressionResolver, - val ismTemplateService: ISMTemplateService -) : TransportMasterNodeAction<PutISMTemplateRequest, PutISMTemplateResponse>( - PutISMTemplateAction.NAME, - transportService, - clusterService, - threadPool, - actionFilters, - Writeable.Reader { PutISMTemplateRequest(it) }, - indexNameExpressionResolver -) { - /** - * callbacks is inexpensive, this value may be - * {@link org.elasticsearch.threadpool.ThreadPool.Names#SAME SAME} (indicating that the callbacks will run on the same thread - * as the cluster state events are fired with) - */ - override fun executor(): String { - return ThreadPool.Names.SAME - } - - override fun read(sin: StreamInput): PutISMTemplateResponse { - return PutISMTemplateResponse(sin) - } - - override fun masterOperation(request: PutISMTemplateRequest, state: ClusterState, listener: ActionListener<PutISMTemplateResponse>) { - ismTemplateService.putISMTemplate(request.templateName, request.ismTemplate, request.masterNodeTimeout(), listener) - } - - override fun checkBlock(request: PutISMTemplateRequest, state: ClusterState): ClusterBlockException? { - return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE) - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt index 455629aaf..b55b61cfd 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt @@ -18,10 +18,7 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanageme import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.optionalTimeField import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ChangePolicy -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplateMetadata import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig -import org.elasticsearch.cluster.metadata.Metadata import org.elasticsearch.common.io.stream.StreamInput import org.elasticsearch.common.io.stream.StreamOutput import org.elasticsearch.common.io.stream.Writeable @@ -38,6 +35,8 @@ const val FAILURES = "failures" const val FAILED_INDICES = "failed_indices" const val UPDATED_INDICES = "updated_indices" +const val ISM_TEMPLATE_FIELD = "policy.ism_template" + fun buildInvalidIndexResponse(builder: XContentBuilder, failedIndices: List<FailedIndex>) { if (failedIndices.isNotEmpty()) { builder.field(FAILURES, true) @@ -95,21 +94,3 @@ fun getPartialChangePolicyBuilder( .endObject() .endObject() } - -/** - * return sorted ism templates map saved in cluster metadata - */ -fun Metadata.ismTemplates(): Map<String, ISMTemplate> { - val ismCustomMetadata: ISMTemplateMetadata? = this.custom(ISMTemplateMetadata.TYPE) - return ismCustomMetadata?.ismTemplates?.toSortedMap() ?: emptyMap() -} - -fun Metadata.Builder.putISMTemplate(name: String, template: ISMTemplate, existing: Map<String, ISMTemplate>): Metadata.Builder { - return this.putCustom(ISMTemplateMetadata.TYPE, - ISMTemplateMetadata(existing.plus(name to template))) -} - -fun Metadata.Builder.removeISMTemplate(name: String, existing: Map<String, ISMTemplate>): Metadata.Builder { - return this.putCustom(ISMTemplateMetadata.TYPE, - ISMTemplateMetadata(existing.minus(name))) -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/util/IndexManagementException.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/util/IndexManagementException.kt new file mode 100644 index 000000000..4f8200db7 --- /dev/null +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/util/IndexManagementException.kt @@ -0,0 +1,45 @@ +package com.amazon.opendistroforelasticsearch.indexmanagement.util + +import org.elasticsearch.ElasticsearchException +import org.elasticsearch.common.Strings +import org.elasticsearch.common.ValidationException +import org.elasticsearch.index.IndexNotFoundException +import org.elasticsearch.rest.RestStatus +import java.lang.IllegalArgumentException + +class IndexManagementException(message: String, val status: RestStatus, ex: Exception) : ElasticsearchException(message, ex) { + + override fun status(): RestStatus { + return status + } + + companion object { + @JvmStatic + fun wrap(ex: Exception): ElasticsearchException { + + var friendlyMsg = ex.message as String + var status = RestStatus.INTERNAL_SERVER_ERROR + when (ex) { + is IndexNotFoundException -> { + status = ex.status() + friendlyMsg = "Configuration index not found" + } + is IllegalArgumentException -> { + status = RestStatus.BAD_REQUEST + friendlyMsg = ex.message as String + } + is ValidationException -> { + status = RestStatus.BAD_REQUEST + friendlyMsg = ex.message as String + } + else -> { + if (!Strings.isNullOrEmpty(ex.message)) { + friendlyMsg = ex.message as String + } + } + } + + return IndexManagementException(friendlyMsg, status, Exception("${ex.javaClass.name}: ${ex.message}")) + } + } +} diff --git a/src/main/resources/mappings/opendistro-ism-config.json b/src/main/resources/mappings/opendistro-ism-config.json index e98c644ef..02365218b 100644 --- a/src/main/resources/mappings/opendistro-ism-config.json +++ b/src/main/resources/mappings/opendistro-ism-config.json @@ -442,6 +442,23 @@ } } } + }, + "ism_template": { + "properties": { + "template_name": { + "type": "keyword" + }, + "index_patterns": { + "type": "keyword" + }, + "priority": { + "type": "long" + }, + "last_updated_time": { + "type": "date", + "format": "strict_date_time||epoch_millis" + } + } } } }, diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt index 1b5a44ec5..a5e2061a1 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt @@ -21,7 +21,6 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlug import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementIndices import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin.Companion.ISM_BASE_URI -import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin.Companion.ISM_TEMPLATE_BASE_URI import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementRestTestCase import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.parseWithType import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ChangePolicy @@ -36,9 +35,6 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagemen import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StateMetaData import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestExplainAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get.GetISMTemplateResponse.Companion.ISM_TEMPLATE -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get.GetISMTemplateResponse.Companion.ISM_TEMPLATES -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get.GetISMTemplateResponse.Companion.TEMPLATE_NAME import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.FAILED_INDICES import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.FAILURES import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.UPDATED_INDICES @@ -703,76 +699,6 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() return true } - protected fun createISMTemplate( - name: String, - template: ISMTemplate - ): Response { - return createISMTemplateJson(name, template.toJsonString()) - } - - protected fun createISMTemplateJson( - name: String, - templateString: String - ): Response { - return client().makeRequest( - "PUT", - "$ISM_TEMPLATE_BASE_URI/$name", - StringEntity(templateString, APPLICATION_JSON) - ) - } - - protected fun getISMTemplatesAsMap(name: String): Map<String, Any> { - val response = client().makeRequest("GET", "$ISM_TEMPLATE_BASE_URI/$name") - assertEquals("Unexpected RestStatus", RestStatus.OK, response.restStatus()) - return response.asMap() - } - - protected fun getISMTemplatesAsObject(name: String?): Map<String, ISMTemplate> { - var endpoint = ISM_TEMPLATE_BASE_URI - if (name != null) endpoint += "/$name" - val response = client().makeRequest("GET", endpoint) - assertEquals("Unable to get template $name", RestStatus.OK, response.restStatus()) - - val xcp = createParser(XContentType.JSON.xContent(), response.entity.content) - val ismTemplates = mutableMapOf<String, ISMTemplate>() - - ensureExpectedToken(Token.START_OBJECT, xcp.nextToken(), xcp) - - while (xcp.nextToken() != Token.END_OBJECT) { - val fieldName = xcp.currentName() - xcp.nextToken() - - when (fieldName) { - ISM_TEMPLATES -> { - ensureExpectedToken(Token.START_ARRAY, xcp.currentToken(), xcp) - - var templateName: String? = null - var template: ISMTemplate? = null - - while (xcp.nextToken() != Token.END_ARRAY) { - when (xcp.currentName()) { - TEMPLATE_NAME -> { - xcp.nextToken() - templateName = xcp.text() - } - ISM_TEMPLATE -> { - // xcp.nextToken() - template = ISMTemplate.parse(xcp) - } - } - if (templateName != null && template != null) { - ismTemplates[templateName] = template - templateName = null - template = null - } - } - } - } - } - - return ismTemplates - } - protected fun assertPredicatesOnISMTemplatesMap( templatePredicates: List<Pair<String, List<Pair<String, (Any?) -> Boolean>>>>, // response map name: predicate response: Map<String, Any?> @@ -792,7 +718,6 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() protected fun assertISMTemplateEquals(expected: ISMTemplate, actualISMTemplateMap: Any?): Boolean { actualISMTemplateMap as Map<String, Any> assertEquals(expected.indexPatterns, actualISMTemplateMap[ISMTemplate.INDEX_PATTERN]) - assertEquals(expected.policyID, actualISMTemplateMap[ISMTemplate.POLICY_ID]) assertEquals(expected.priority, actualISMTemplateMap[ISMTemplate.PRIORITY]) return true } @@ -801,13 +726,8 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() assertNotNull(actual) if (actual != null) { assertEquals(expected.indexPatterns, actual.indexPatterns) - assertEquals(expected.policyID, actual.policyID) assertEquals(expected.priority, actual.priority) } return true } - - protected fun deleteISMTemplate(name: String): Response { - return client().makeRequest("DELETE", "$ISM_TEMPLATE_BASE_URI/$name") - } } diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/TestHelpers.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/TestHelpers.kt index 37e3557c6..67097dc71 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/TestHelpers.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/TestHelpers.kt @@ -67,10 +67,11 @@ fun randomPolicy( schemaVersion: Long = ESRestTestCase.randomLong(), lastUpdatedTime: Instant = Instant.now().truncatedTo(ChronoUnit.MILLIS), errorNotification: ErrorNotification? = randomErrorNotification(), - states: List<State> = List(ESRestTestCase.randomIntBetween(1, 10)) { randomState() } + states: List<State> = List(ESRestTestCase.randomIntBetween(1, 10)) { randomState() }, + ismTemplate: ISMTemplate? = null ): Policy { return Policy(id = id, schemaVersion = schemaVersion, lastUpdatedTime = lastUpdatedTime, - errorNotification = errorNotification, defaultState = states[0].name, states = states, description = description) + errorNotification = errorNotification, defaultState = states[0].name, states = states, description = description, ismTemplate = ismTemplate) } fun randomState( @@ -317,13 +318,11 @@ fun randomSweptManagedIndexConfig( fun randomISMTemplate( indexPatterns: List<String> = listOf(ESRestTestCase.randomAlphaOfLength(10) + "*"), - policyID: String = ESRestTestCase.randomAlphaOfLength(10), - priority: Int = ESRestTestCase.randomIntBetween(0, 200), + priority: Int = ESRestTestCase.randomIntBetween(0, 100), lastUpdatedTime: Instant = Instant.now().truncatedTo(ChronoUnit.MILLIS) ): ISMTemplate { return ISMTemplate( indexPatterns = indexPatterns, - policyID = policyID, priority = priority, lastUpdatedTime = lastUpdatedTime ) diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt index 1c2287a7c..5b6f39cde 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt @@ -1,21 +1,18 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler -import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin.Companion.ISM_TEMPLATE_BASE_URI import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.Policy import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.State import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.action.ReadOnlyActionConfig import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.randomErrorNotification +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.randomPolicy import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings -import com.amazon.opendistroforelasticsearch.indexmanagement.makeRequest import com.amazon.opendistroforelasticsearch.indexmanagement.randomInstant import com.amazon.opendistroforelasticsearch.indexmanagement.waitFor import org.elasticsearch.client.ResponseException import org.elasticsearch.common.settings.Settings import org.elasticsearch.rest.RestStatus -import org.elasticsearch.rest.RestRequest.Method.GET -import org.junit.After import java.time.Instant import java.time.temporal.ChronoUnit import java.util.Locale @@ -24,104 +21,33 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { private val testIndexName = javaClass.simpleName.toLowerCase(Locale.ROOT) - private val templateName = "t1" - private val templateName2 = "t2" - - @After - fun `clean template`() { - deleteISMTemplate(templateName) - deleteISMTemplate(templateName2) - } - - fun `test ISM template`() { - val ismTemp = ISMTemplate(listOf("log*"), "policy_1", 100, randomInstant()) - - var res = createISMTemplate(templateName, ismTemp) - assertEquals("Unable to create new ISM template", RestStatus.CREATED, res.restStatus()) - - res = createISMTemplate(templateName, ismTemp) - assertEquals("Unable to update new ISM template", RestStatus.OK, res.restStatus()) - - var getRes = getISMTemplatesAsObject(templateName) - assertISMTemplateEquals(ismTemp, getRes[templateName]) - - val ismTemp2 = ISMTemplate(listOf("trace*"), "policy_1", 100, randomInstant()) - createISMTemplate(templateName2, ismTemp2) - getRes = getISMTemplatesAsObject("$templateName,$templateName2") - val getRes2 = getISMTemplatesAsObject(null) - assertEquals(getRes, getRes2) - assertISMTemplateEquals(ismTemp, getRes[templateName]) - assertISMTemplateEquals(ismTemp2, getRes[templateName2]) - - val delRes = deleteISMTemplate(templateName) - assertEquals(true, delRes.asMap()["acknowledged"]) - } - - fun `test get not exist template`() { - try { - client().makeRequest(GET.toString(), "$ISM_TEMPLATE_BASE_URI/$templateName") - fail("Expect a failure") - } catch (e: ResponseException) { - assertEquals("Unexpected RestStatus", RestStatus.NOT_FOUND, e.response.restStatus()) - val actualMessage = e.response.asMap() - val expectErrorMessage = mapOf( - "error" to mapOf( - "root_cause" to listOf<Map<String, Any>>( - mapOf("type" to "resource_not_found_exception", "reason" to "index template matching [$templateName] not found") - ), - "type" to "resource_not_found_exception", - "reason" to "index template matching [$templateName] not found" - ), - "status" to 404 - ) - assertEquals(expectErrorMessage, actualMessage) - } - } + private val policyID1 = "t1" + private val policyID2 = "t2" fun `test add template with invalid index pattern`() { try { - val ismTemp = ISMTemplate(listOf(" "), "policy_1", 100, randomInstant()) - createISMTemplate(templateName, ismTemp) + val ismTemp = ISMTemplate(listOf(" "), 100, randomInstant()) + createPolicy(randomPolicy(ismTemplate = ismTemp), policyID1) fail("Expect a failure") } catch (e: ResponseException) { assertEquals("Unexpected RestStatus", RestStatus.BAD_REQUEST, e.response.restStatus()) - val actualMessage = e.response.asMap() - val expectErrorMessage = mapOf( - "error" to mapOf( - "reason" to "index_template [$templateName] invalid, cause [Validation Failed: 1: index_patterns [ ] must not contain a space;2: index_pattern [ ] must not contain the following characters [ , \", *, \\, <, |, ,, >, /, ?];]", - "type" to "invalid_index_template_exception", - "root_cause" to listOf<Map<String, Any>>( - mapOf("reason" to "index_template [$templateName] invalid, cause [Validation Failed: 1: index_patterns [ ] must not contain a space;2: index_pattern [ ] must not contain the following characters [ , \", *, \\, <, |, ,, >, /, ?];]", - "type" to "invalid_index_template_exception") - ) - ), - "status" to 400 - ) - assertEquals(expectErrorMessage, actualMessage) + val actualMessage = e.response.asMap()["error"] as Map<String, Any> + val expectedReason = "Validation Failed: 1: index_patterns [ ] must not contain a space;2: index_pattern [ ] must not contain the following characters [ , \", *, \\, <, |, ,, >, /, ?];" + assertEquals(expectedReason, actualMessage["reason"]) } } fun `test add template with overlapping index pattern`() { try { - val ismTemp = ISMTemplate(listOf("log*"), "policy_1", 100, randomInstant()) - createISMTemplate(templateName, ismTemp) - createISMTemplate(templateName2, ismTemp) + val ismTemp = ISMTemplate(listOf("log*"), 100, randomInstant()) + createPolicy(randomPolicy(ismTemplate = ismTemp), policyID1) + createPolicy(randomPolicy(ismTemplate = ismTemp), policyID2) fail("Expect a failure") } catch (e: ResponseException) { assertEquals("Unexpected RestStatus", RestStatus.BAD_REQUEST, e.response.restStatus()) - val actualMessage = e.response.asMap() - val expectErrorMessage = mapOf( - "error" to mapOf( - "reason" to "new ism template $templateName2 has index pattern [log*] matching existing templates t1 => [log*], please use a different priority than 100", - "type" to "illegal_argument_exception", - "root_cause" to listOf<Map<String, Any>>( - mapOf("reason" to "new ism template $templateName2 has index pattern [log*] matching existing templates t1 => [log*], please use a different priority than 100", - "type" to "illegal_argument_exception") - ) - ), - "status" to 400 - ) - assertEquals(expectErrorMessage, actualMessage) + val actualMessage = e.response.asMap()["error"] as Map<String, Any> + val expectedReason = "new policy $policyID2 has an ism template with index pattern [log*] matching existing templates $policyID1 => [log*], please use a different priority than 100" + assertEquals(expectedReason, actualMessage["reason"]) } } @@ -134,8 +60,7 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { // need to specify policyID null, can remove after policyID deprecated createIndex(indexName1, null) - val ismTemp = ISMTemplate(listOf("log*"), policyID, 100, randomInstant()) - createISMTemplate(templateName, ismTemp) + val ismTemp = ISMTemplate(listOf("log*"), 100, randomInstant()) val actionConfig = ReadOnlyActionConfig(0) val states = listOf( @@ -148,7 +73,8 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { lastUpdatedTime = Instant.now().truncatedTo(ChronoUnit.MILLIS), errorNotification = randomErrorNotification(), defaultState = states[0].name, - states = states + states = states, + ismTemplate = ismTemp ) createPolicy(policy, policyID) diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ActionTests.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ActionTests.kt index f5a9459e9..003329b8f 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ActionTests.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ActionTests.kt @@ -21,9 +21,6 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagemen import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.indexpolicy.IndexPolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.explain.ExplainAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.getpolicy.GetPolicyAction -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.delete.DeleteISMTemplateAction -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get.GetISMTemplateAction -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put.PutISMTemplateAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.removepolicy.RemovePolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.retryfailedmanagedindex.RetryFailedManagedIndexAction import org.elasticsearch.test.ESTestCase @@ -68,19 +65,4 @@ class ActionTests : ESTestCase() { assertNotNull(GetPolicyAction.NAME) assertEquals(GetPolicyAction.INSTANCE.name(), GetPolicyAction.NAME) } - - fun `test get template action name`() { - assertNotNull(GetISMTemplateAction.NAME) - assertEquals(GetISMTemplateAction.INSTANCE.name(), GetISMTemplateAction.NAME) - } - - fun `test put template action name`() { - assertNotNull(PutISMTemplateAction.NAME) - assertEquals(PutISMTemplateAction.INSTANCE.name(), PutISMTemplateAction.NAME) - } - - fun `test delete template action name`() { - assertNotNull(DeleteISMTemplateAction.NAME) - assertEquals(DeleteISMTemplateAction.INSTANCE.name(), DeleteISMTemplateAction.NAME) - } } diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequestTests.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequestTests.kt deleted file mode 100644 index 260cae0ee..000000000 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequestTests.kt +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.delete - -import org.elasticsearch.common.io.stream.BytesStreamOutput -import org.elasticsearch.common.io.stream.StreamInput -import org.elasticsearch.test.ESTestCase - -class DeleteISMTemplateRequestTests : ESTestCase() { - - fun `test delete template request`() { - val templateName = "t1" - val req = DeleteISMTemplateRequest(templateName) - - val out = BytesStreamOutput() - req.writeTo(out) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newReq = DeleteISMTemplateRequest(sin) - assertEquals(templateName, newReq.templateName) - } -} diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequestTests.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequestTests.kt deleted file mode 100644 index 68a8e3721..000000000 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequestTests.kt +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get - -import org.elasticsearch.common.io.stream.BytesStreamOutput -import org.elasticsearch.common.io.stream.StreamInput -import org.elasticsearch.test.ESTestCase - -class GetISMTemplateRequestTests : ESTestCase() { - - fun `test get templates request`() { - val templateNames = arrayOf("t1") - val req = GetISMTemplateRequest(templateNames) - - val out = BytesStreamOutput() - req.writeTo(out) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newReq = GetISMTemplateRequest(sin) - assertEquals(templateNames, newReq.templateNames) - } -} diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponseTests.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponseTests.kt deleted file mode 100644 index e55ad8ec8..000000000 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponseTests.kt +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get - -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate -import com.amazon.opendistroforelasticsearch.indexmanagement.randomInstant -import org.elasticsearch.common.io.stream.BytesStreamOutput -import org.elasticsearch.common.io.stream.StreamInput -import org.elasticsearch.test.ESTestCase - -class GetISMTemplateResponseTests : ESTestCase() { - - fun `test get templates response`() { - val ismTemplates = mapOf("t1" to ISMTemplate(listOf("log*"), "policy_1", 100, randomInstant())) - val req = GetISMTemplateResponse(ismTemplates) - - val out = BytesStreamOutput() - req.writeTo(out) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newReq = GetISMTemplateResponse(sin) - assertEquals(ismTemplates, newReq.ismTemplates) - } -} diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateRequestTests.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateRequestTests.kt deleted file mode 100644 index b8747eb4e..000000000 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateRequestTests.kt +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put - -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate -import com.amazon.opendistroforelasticsearch.indexmanagement.randomInstant -import org.elasticsearch.common.io.stream.BytesStreamOutput -import org.elasticsearch.common.io.stream.StreamInput -import org.elasticsearch.test.ESTestCase - -class PutISMTemplateRequestTests : ESTestCase() { - - fun `test put template request`() { - val templateName = "t1" - val ismTemplate = ISMTemplate(listOf("log*"), "policy_1", 100, randomInstant()) - val req = PutISMTemplateRequest(templateName, ismTemplate) - - val out = BytesStreamOutput() - req.writeTo(out) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newReq = PutISMTemplateRequest(sin) - assertEquals(templateName, newReq.templateName) - assertEquals(ismTemplate, newReq.ismTemplate) - } -} \ No newline at end of file diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponseTests.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponseTests.kt deleted file mode 100644 index 3d9f25c97..000000000 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponseTests.kt +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put - -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate -import com.amazon.opendistroforelasticsearch.indexmanagement.randomInstant -import org.elasticsearch.common.io.stream.BytesStreamOutput -import org.elasticsearch.common.io.stream.StreamInput -import org.elasticsearch.rest.RestStatus -import org.elasticsearch.test.ESTestCase - -class PutISMTemplateResponseTests : ESTestCase() { - - fun `test put template response`() { - val id = "t1" - val template = ISMTemplate(listOf("log*"), "policy_1", 100, randomInstant()) - val status = RestStatus.OK - val req = PutISMTemplateResponse(id, template, status) - - val out = BytesStreamOutput() - req.writeTo(out) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newReq = PutISMTemplateResponse(sin) - assertEquals(id, newReq.id) - assertEquals(template, newReq.template) - assertEquals(status, newReq.status) - } -} diff --git a/src/test/resources/mappings/cached-opendistro-ism-config.json b/src/test/resources/mappings/cached-opendistro-ism-config.json index e98c644ef..02365218b 100644 --- a/src/test/resources/mappings/cached-opendistro-ism-config.json +++ b/src/test/resources/mappings/cached-opendistro-ism-config.json @@ -442,6 +442,23 @@ } } } + }, + "ism_template": { + "properties": { + "template_name": { + "type": "keyword" + }, + "index_patterns": { + "type": "keyword" + }, + "priority": { + "type": "long" + }, + "last_updated_time": { + "type": "date", + "format": "strict_date_time||epoch_millis" + } + } } } }, From 8c7067ce95ddb5c842cdaec528c575478d34dfe9 Mon Sep 17 00:00:00 2001 From: bowenlan-amzn <bowenlan@amazon.com> Date: Mon, 11 Jan 2021 19:58:25 -0800 Subject: [PATCH 15/21] new implementation --- .../indexmanagement/IndexManagementPlugin.kt | 39 +---- .../elasticapi/ElasticExtensions.kt | 9 ++ .../ISMTemplateService.kt | 140 ++---------------- .../ManagedIndexCoordinator.kt | 44 +++++- .../elasticapi/ElasticExtensions.kt | 31 ++++ .../indexstatemanagement/model/ISMTemplate.kt | 26 +--- .../model/ISMTemplateMetadata.kt | 102 ------------- .../indexstatemanagement/model/Policy.kt | 18 ++- .../resthandler/RestAddISMTemplateAction.kt | 74 --------- .../RestDeleteISMTemplateAction.kt | 53 ------- .../resthandler/RestGetISMTemplateAction.kt | 54 ------- .../indexpolicy/TransportIndexPolicyAction.kt | 56 ++++++- .../delete/DeleteISMTemplateAction.kt | 26 ---- .../delete/DeleteISMTemplateRequest.kt | 48 ------ .../TransportDeleteISMTemplateAction.kt | 68 --------- .../ismtemplate/get/GetISMTemplateAction.kt | 25 ---- .../ismtemplate/get/GetISMTemplateRequest.kt | 47 ------ .../ismtemplate/get/GetISMTemplateResponse.kt | 52 ------- .../get/TransportGetISMTemplateAction.kt | 74 --------- .../ismtemplate/put/PutISMTemplateAction.kt | 25 ---- .../ismtemplate/put/PutISMTemplateRequest.kt | 54 ------- .../ismtemplate/put/PutISMTemplateResponse.kt | 66 --------- .../put/TransportPutISMTemplateAction.kt | 72 --------- .../util/RestHandlerUtils.kt | 23 +-- .../util/IndexManagementException.kt | 45 ++++++ .../mappings/opendistro-ism-config.json | 17 +++ .../IndexStateManagementRestTestCase.kt | 80 ---------- .../indexstatemanagement/TestHelpers.kt | 9 +- .../resthandler/ISMTemplateRestAPIIT.kt | 108 +++----------- .../transport/action/ActionTests.kt | 18 --- .../delete/DeleteISMTemplateRequestTests.kt | 34 ----- .../get/GetISMTemplateRequestTests.kt | 34 ----- .../get/GetISMTemplateResponseTests.kt | 36 ----- .../put/PutISMTemplateRequestTests.kt | 38 ----- .../put/PutISMTemplateResponseTests.kt | 41 ----- .../cached-opendistro-ism-config.json | 17 +++ 36 files changed, 265 insertions(+), 1438 deletions(-) delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateMetadata.kt delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestAddISMTemplateAction.kt delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestDeleteISMTemplateAction.kt delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestGetISMTemplateAction.kt delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateAction.kt delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequest.kt delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/TransportDeleteISMTemplateAction.kt delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateAction.kt delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequest.kt delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponse.kt delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/TransportGetISMTemplateAction.kt delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateAction.kt delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateRequest.kt delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponse.kt delete mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/TransportPutISMTemplateAction.kt create mode 100644 src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/util/IndexManagementException.kt delete mode 100644 src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequestTests.kt delete mode 100644 src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequestTests.kt delete mode 100644 src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponseTests.kt delete mode 100644 src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateRequestTests.kt delete mode 100644 src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponseTests.kt diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/IndexManagementPlugin.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/IndexManagementPlugin.kt index e9f57759e..ae31e6055 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/IndexManagementPlugin.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/IndexManagementPlugin.kt @@ -18,18 +18,14 @@ package com.amazon.opendistroforelasticsearch.indexmanagement import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.IndexStateManagementHistory import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ManagedIndexCoordinator import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ManagedIndexRunner -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplateMetadata import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.updateindexmetadata.TransportUpdateManagedIndexMetaDataAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.updateindexmetadata.UpdateManagedIndexMetaDataAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.Policy -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestAddISMTemplateAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestAddPolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestChangePolicyAction -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestDeleteISMTemplateAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestDeletePolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestExplainAction -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestGetISMTemplateAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestGetPolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestIndexPolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestRemovePolicyAction @@ -47,12 +43,6 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagemen import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.getpolicy.TransportGetPolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.indexpolicy.IndexPolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.indexpolicy.TransportIndexPolicyAction -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.delete.DeleteISMTemplateAction -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.delete.TransportDeleteISMTemplateAction -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get.GetISMTemplateAction -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get.TransportGetISMTemplateAction -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put.PutISMTemplateAction -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put.TransportPutISMTemplateAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.removepolicy.RemovePolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.removepolicy.TransportRemovePolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.retryfailedmanagedindex.RetryFailedManagedIndexAction @@ -100,13 +90,10 @@ import org.elasticsearch.action.ActionRequest import org.elasticsearch.action.ActionResponse import org.elasticsearch.action.support.ActionFilter import org.elasticsearch.client.Client -import org.elasticsearch.cluster.NamedDiff import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver -import org.elasticsearch.cluster.metadata.Metadata import org.elasticsearch.cluster.node.DiscoveryNodes import org.elasticsearch.cluster.service.ClusterService import org.elasticsearch.common.io.stream.NamedWriteableRegistry -import org.elasticsearch.common.io.stream.Writeable import org.elasticsearch.common.settings.ClusterSettings import org.elasticsearch.common.settings.IndexScopedSettings import org.elasticsearch.common.settings.Setting @@ -145,7 +132,6 @@ internal class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, Act const val ROLLUP_BASE_URI = "$OPEN_DISTRO_BASE_URI/_rollup" const val POLICY_BASE_URI = "$ISM_BASE_URI/policies" const val ROLLUP_JOBS_BASE_URI = "$ROLLUP_BASE_URI/jobs" - const val ISM_TEMPLATE_BASE_URI = "$ISM_BASE_URI/templates" const val INDEX_MANAGEMENT_INDEX = ".opendistro-ism-config" const val INDEX_MANAGEMENT_JOB_TYPE = "opendistro-index-management" const val INDEX_STATE_MANAGEMENT_HISTORY_TYPE = "managed_index_meta_data" @@ -211,10 +197,7 @@ internal class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, Act RestIndexRollupAction(), RestStartRollupAction(), RestStopRollupAction(), - RestExplainRollupAction(), - RestAddISMTemplateAction(), - RestGetISMTemplateAction(), - RestDeleteISMTemplateAction() + RestExplainRollupAction() ) } @@ -315,25 +298,7 @@ internal class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, Act ActionPlugin.ActionHandler(StartRollupAction.INSTANCE, TransportStartRollupAction::class.java), ActionPlugin.ActionHandler(StopRollupAction.INSTANCE, TransportStopRollupAction::class.java), ActionPlugin.ActionHandler(ExplainRollupAction.INSTANCE, TransportExplainRollupAction::class.java), - ActionPlugin.ActionHandler(UpdateRollupMappingAction.INSTANCE, TransportUpdateRollupMappingAction::class.java), - ActionPlugin.ActionHandler(PutISMTemplateAction.INSTANCE, TransportPutISMTemplateAction::class.java), - ActionPlugin.ActionHandler(GetISMTemplateAction.INSTANCE, TransportGetISMTemplateAction::class.java), - ActionPlugin.ActionHandler(DeleteISMTemplateAction.INSTANCE, TransportDeleteISMTemplateAction::class.java) - ) - } - - override fun getNamedWriteables(): List<NamedWriteableRegistry.Entry> { - return listOf( - NamedWriteableRegistry.Entry( - Metadata.Custom::class.java, - ISMTemplateMetadata.TYPE, - Writeable.Reader { sin -> ISMTemplateMetadata(sin) } - ), - NamedWriteableRegistry.Entry( - NamedDiff::class.java, - ISMTemplateMetadata.TYPE, - Writeable.Reader { sin -> ISMTemplateMetadata.readDiffFrom(sin) } - ) + ActionPlugin.ActionHandler(UpdateRollupMappingAction.INSTANCE, TransportUpdateRollupMappingAction::class.java) ) } diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/elasticapi/ElasticExtensions.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/elasticapi/ElasticExtensions.kt index d0dab01d2..735ba75ca 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/elasticapi/ElasticExtensions.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/elasticapi/ElasticExtensions.kt @@ -17,6 +17,8 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.Policy import com.amazon.opendistroforelasticsearch.indexmanagement.util.NO_ID import com.amazon.opendistroforelasticsearch.jobscheduler.spi.utils.LockService import kotlinx.coroutines.delay @@ -70,6 +72,13 @@ fun XContentBuilder.optionalTimeField(name: String, instant: Instant?): XContent return this.timeField(name, "${name}_in_millis", instant.toEpochMilli()) } +fun XContentBuilder.optionalISMTemplateField(name: String, ismTemplate: ISMTemplate?): XContentBuilder { + if (ismTemplate == null) { + return nullField(name) + } + return this.field(Policy.ISM_TEMPLATE, ismTemplate) +} + /** * Retries the given [block] of code as specified by the receiver [BackoffPolicy], * if [block] throws an [ElasticsearchException] that is retriable (502, 503, 504). diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt index 336a0ee70..a8d5849f7 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt @@ -15,118 +15,20 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement -import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.filterNotNullValues import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put.PutISMTemplateResponse -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.ismTemplates -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.putISMTemplate -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.removeISMTemplate +import com.amazon.opendistroforelasticsearch.indexmanagement.util.IndexManagementException import org.apache.logging.log4j.LogManager import org.apache.lucene.util.automaton.Operations -import org.elasticsearch.action.ActionListener -import org.elasticsearch.action.support.master.AcknowledgedResponse -import org.elasticsearch.cluster.ClusterState -import org.elasticsearch.cluster.ClusterStateUpdateTask +import org.elasticsearch.ElasticsearchException import org.elasticsearch.cluster.metadata.IndexMetadata -import org.elasticsearch.cluster.metadata.Metadata -import org.elasticsearch.cluster.service.ClusterService -import org.elasticsearch.common.Priority import org.elasticsearch.common.Strings import org.elasticsearch.common.ValidationException -import org.elasticsearch.common.inject.Inject import org.elasticsearch.common.regex.Regex -import org.elasticsearch.common.unit.TimeValue -import org.elasticsearch.indices.InvalidIndexTemplateException -import org.elasticsearch.rest.RestStatus -import java.util.stream.Collectors -import java.util.Locale private val log = LogManager.getLogger(ISMTemplateService::class.java) -class ISMTemplateService @Inject constructor( - val clusterService: ClusterService -) { - /** - * save ISM template to cluster state metadata - */ - fun putISMTemplate( - templateName: String, - template: ISMTemplate, - masterTimeout: TimeValue, - listener: ActionListener<PutISMTemplateResponse> - ) { - clusterService.submitStateUpdateTask( - IndexManagementPlugin.PLUGIN_NAME, - object : ClusterStateUpdateTask(Priority.NORMAL) { - override fun execute(currentState: ClusterState): ClusterState { - return addISMTemplate(currentState, templateName, template) - } - - override fun onFailure(source: String, e: Exception) { - listener.onFailure(e) - } - - override fun timeout(): TimeValue = masterTimeout - - override fun clusterStateProcessed(source: String, oldState: ClusterState, newState: ClusterState) { - var status = RestStatus.CREATED - val oldTemplate = oldState.metadata.ismTemplates()[templateName] - if (oldTemplate != null) { - status = RestStatus.OK - } - listener.onResponse(PutISMTemplateResponse(templateName, template, status)) - } - } - ) - } - - fun addISMTemplate(currentState: ClusterState, templateName: String, template: ISMTemplate): ClusterState { - val existingTemplates = currentState.metadata.ismTemplates() - val existingTemplate = existingTemplates[templateName] - - if (template == existingTemplate) return currentState - - // find templates with overlapping index pattern - val overlaps = findConflictingISMTemplates(templateName, template.indexPatterns, template.priority, existingTemplates) - if (overlaps.isNotEmpty()) { - val esg = "new ism template $templateName has index pattern ${template.indexPatterns} " + - "matching existing templates ${overlaps.entries.stream().map { "${it.key} => ${it.value}" }.collect(Collectors.joining(","))}," + - " please use a different priority than ${template.priority}" - throw IllegalArgumentException(esg) - } - - validateFormat(templateName, template.indexPatterns) - - return ClusterState.builder(currentState).metadata(Metadata.builder(currentState.metadata()) - .putISMTemplate(templateName, template, existingTemplates)).build() - } - - /** - * remove ISM template from cluster state metadata - */ - fun deleteISMTemplate(templateName: String, masterTimeout: TimeValue, listener: ActionListener<AcknowledgedResponse>) { - clusterService.submitStateUpdateTask( - IndexManagementPlugin.PLUGIN_NAME, - object : ClusterStateUpdateTask(Priority.NORMAL) { - override fun execute(currentState: ClusterState): ClusterState { - val existingTemplates = currentState.metadata.ismTemplates() - return ClusterState.builder(currentState).metadata(Metadata.builder(currentState.metadata) - .removeISMTemplate(templateName, existingTemplates)).build() - } - - override fun onFailure(source: String, e: Exception) { - listener.onFailure(e) - } - - override fun timeout(): TimeValue = masterTimeout - - override fun clusterStateProcessed(source: String, oldState: ClusterState, newState: ClusterState) { - listener.onResponse(AcknowledgedResponse(true)) - } - } - ) - } - +class ISMTemplateService { companion object { /** * find the matching template for the index @@ -136,7 +38,7 @@ class ISMTemplateService @Inject constructor( * * @param ismTemplates current ISM templates saved in metadata * @param indexMetadata cluster state index metadata - * @return template name matching with given index + * @return policyID */ @Suppress("ReturnCount") fun findMatchingISMTemplate(ismTemplates: Map<String, ISMTemplate>, indexMetadata: IndexMetadata): String? { @@ -172,26 +74,8 @@ class ISMTemplateService @Inject constructor( * reusing ES validate function in MetadataIndexTemplateService */ @Suppress("ComplexMethod") - fun validateFormat(templateName: String, indexPatterns: List<String>) { + fun validateFormat(indexPatterns: List<String>): ElasticsearchException? { val validationErrors = mutableListOf<String>() - if (templateName.contains(" ")) { - validationErrors.add("name must not contain a space") - } - if (templateName.contains(",")) { - validationErrors.add("name must not contain a ','") - } - if (templateName.contains("#")) { - validationErrors.add("name must not contain a '#'") - } - if (templateName.contains("*")) { - validationErrors.add("name must not contain a '*'") - } - if (templateName.startsWith("_")) { - validationErrors.add("name must not start with '_'") - } - if (templateName.toLowerCase(Locale.ROOT) != templateName) { - validationErrors.add("name must be lower cased") - } for (indexPattern in indexPatterns) { if (indexPattern.contains(" ")) { validationErrors.add("index_patterns [$indexPattern] must not contain a space") @@ -217,8 +101,9 @@ class ISMTemplateService @Inject constructor( if (validationErrors.size > 0) { val validationException = ValidationException() validationException.addValidationErrors(validationErrors) - throw InvalidIndexTemplateException(templateName, validationException.message) + return IndexManagementException.wrap(validationException) } + return null } /** @@ -231,17 +116,18 @@ class ISMTemplateService @Inject constructor( candidate: String, indexPatterns: List<String>, priority: Int, - ismTemplates: Map<String, ISMTemplate> + ismTemplates: Map<String, ISMTemplate?> ): Map<String, List<String>> { val automaton1 = Regex.simpleMatchToAutomaton(*indexPatterns.toTypedArray()) val overlappingTemplates = mutableMapOf<String, List<String>>() // focus on template with same priority - ismTemplates.filter { it.value.priority == priority }.forEach { (templateName, template) -> + ismTemplates.filterNotNullValues() + .filter { it.value.priority == priority }.forEach { (policyID, template) -> val automaton2 = Regex.simpleMatchToAutomaton(*template.indexPatterns.toTypedArray()) if (!Operations.isEmpty(Operations.intersection(automaton1, automaton2))) { - log.info("existing template $templateName overlaps candidate $candidate") - overlappingTemplates[templateName] = template.indexPatterns + log.info("existing ism_template in $policyID overlaps candidate $candidate") + overlappingTemplates[policyID] = template.indexPatterns } } overlappingTemplates.remove(candidate) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt index ba6a477dd..bd621050b 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt @@ -17,6 +17,7 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanageme import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementIndices +import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.parseWithType import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ISMTemplateService.Companion.findMatchingISMTemplate import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.getClusterStateManagedIndexConfig @@ -24,9 +25,12 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagemen import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.getPolicyID import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.retry import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.suspendUntil +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.filterNotNullValues +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.ismTemplatesFromSearchResponse import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.shouldCreateManagedIndexConfig import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.shouldDeleteManagedIndexConfig import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.shouldDeleteManagedIndexMetaData +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.coordinator.ClusterStateManagedIndexConfig import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData @@ -39,6 +43,7 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagemen import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings.Companion.SWEEP_PERIOD import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.updateindexmetadata.UpdateManagedIndexMetaDataAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.updateindexmetadata.UpdateManagedIndexMetaDataRequest +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.ISM_TEMPLATE_FIELD import com.amazon.opendistroforelasticsearch.indexmanagement.util.OpenForTesting import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.managedIndexConfigIndexRequest import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.deleteManagedIndexRequest @@ -47,7 +52,6 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagemen import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.getSweptManagedIndexSearchRequest import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.isFailed import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.isPolicyCompleted -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.ismTemplates import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.updateEnableManagedIndexRequest import com.amazon.opendistroforelasticsearch.indexmanagement.util.NO_ID import kotlinx.coroutines.CoroutineName @@ -63,6 +67,7 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse import org.elasticsearch.action.bulk.BackoffPolicy import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.search.SearchRequest import org.elasticsearch.action.search.SearchResponse import org.elasticsearch.action.support.IndicesOptions import org.elasticsearch.action.support.master.AcknowledgedResponse @@ -70,6 +75,7 @@ import org.elasticsearch.client.Client import org.elasticsearch.cluster.ClusterChangedEvent import org.elasticsearch.cluster.ClusterState import org.elasticsearch.cluster.ClusterStateListener +import org.elasticsearch.cluster.block.ClusterBlockException import org.elasticsearch.cluster.service.ClusterService import org.elasticsearch.common.bytes.BytesReference import org.elasticsearch.common.component.LifecycleListener @@ -81,7 +87,10 @@ import org.elasticsearch.common.xcontent.XContentHelper import org.elasticsearch.common.xcontent.XContentParser import org.elasticsearch.common.xcontent.XContentType import org.elasticsearch.index.Index +import org.elasticsearch.index.IndexNotFoundException +import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.rest.RestStatus +import org.elasticsearch.search.builder.SearchSourceBuilder import org.elasticsearch.threadpool.Scheduler import org.elasticsearch.threadpool.ThreadPool @@ -291,26 +300,45 @@ class ManagedIndexCoordinator( /** * build requests to create jobs for indices matching ISM templates */ - fun getMatchingIndicesUpdateReqs(clusterState: ClusterState, indexNames: List<String>): List<DocWriteRequest<*>> { + suspend fun getMatchingIndicesUpdateReqs(clusterState: ClusterState, indexNames: List<String>): List<DocWriteRequest<*>> { val indexMetadatas = clusterState.metadata.indices - val templates = clusterState.metadata.ismTemplates() + val templates = getISMTemplates() - val indexToMatchMap = indexNames.map { indexName -> + val indexToMatchedPolicy = indexNames.map { indexName -> indexName to findMatchingISMTemplate(templates, indexMetadatas[indexName]) }.toMap() val updateManagedIndexReqs = mutableListOf<DocWriteRequest<*>>() - indexToMatchMap.filter { (_, template) -> template != null }.forEach { (index, template) -> + indexToMatchedPolicy.filterNotNullValues() + .forEach { (index, policyID) -> val indexUuid = indexMetadatas[index].indexUUID - val policyID = templates[template]?.policyID - if (indexUuid != null && policyID != null) { - updateManagedIndexReqs.add(managedIndexConfigIndexRequest(index, indexUuid, policyID, jobInterval)) + if (indexUuid != null) { + logger.info("auto manage index $index to policy $policyID") + updateManagedIndexReqs.add( + managedIndexConfigIndexRequest(index, indexUuid, policyID, jobInterval)) } } return updateManagedIndexReqs } + suspend fun getISMTemplates(): Map<String, ISMTemplate> { + val searchRequest = SearchRequest() + .source( + SearchSourceBuilder().query( + QueryBuilders.existsQuery(ISM_TEMPLATE_FIELD))) + .indices(INDEX_MANAGEMENT_INDEX) + + return try { + val response: SearchResponse = client.suspendUntil { search(searchRequest, it) } + ismTemplatesFromSearchResponse(response).filterNotNullValues() + } catch (ex: IndexNotFoundException) { + emptyMap() + } catch (ex: ClusterBlockException) { + emptyMap() + } + } + /** * Background sweep process that periodically sweeps for updates to ManagedIndices * diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/elasticapi/ElasticExtensions.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/elasticapi/ElasticExtensions.kt index ab2d32b28..e7bc7ca69 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/elasticapi/ElasticExtensions.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/elasticapi/ElasticExtensions.kt @@ -17,10 +17,18 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi +import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.parseWithType +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.Policy import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.coordinator.ClusterStateManagedIndexConfig import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings +import org.elasticsearch.action.search.SearchResponse import org.elasticsearch.cluster.metadata.IndexMetadata +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler +import org.elasticsearch.common.xcontent.NamedXContentRegistry +import org.elasticsearch.common.xcontent.XContentFactory +import org.elasticsearch.common.xcontent.XContentType /** * Compares current and previous IndexMetaData to determine if we should create [ManagedIndexConfig]. @@ -98,3 +106,26 @@ fun IndexMetadata.getManagedIndexMetaData(): ManagedIndexMetaData? { } return null } + +/** + * Do a exists search query to retrieve all policy with ism_template field + * parse search response with this function + * + * @return map of policyID to ISMTemplate in this policy + */ +fun ismTemplatesFromSearchResponse(response: SearchResponse, xContentRegistry: NamedXContentRegistry = NamedXContentRegistry.EMPTY): + Map<String, ISMTemplate?> { + return response.hits.hits.map { + val id = it.id + val seqNo = it.seqNo + val primaryTerm = it.primaryTerm + val xcp = XContentFactory.xContent(XContentType.JSON) + .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, it.sourceAsString) + xcp.parseWithType(id, seqNo, primaryTerm, Policy.Companion::parse) + .copy(id = id, seqNo = seqNo, primaryTerm = primaryTerm) + }.map { it.id to it.ismTemplate }.toMap() +} + +@Suppress("UNCHECKED_CAST") +fun <K, V> Map<K, V?>.filterNotNullValues(): Map<K, V> = + filterValues { it != null } as Map<K, V> diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplate.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplate.kt index dcbed958d..0173459b7 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplate.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplate.kt @@ -18,10 +18,9 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanageme import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.instant import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.optionalTimeField import org.apache.logging.log4j.LogManager -import org.elasticsearch.cluster.AbstractDiffable -import org.elasticsearch.cluster.Diff import org.elasticsearch.common.io.stream.StreamInput import org.elasticsearch.common.io.stream.StreamOutput +import org.elasticsearch.common.io.stream.Writeable import org.elasticsearch.common.xcontent.ToXContent import org.elasticsearch.common.xcontent.ToXContentObject import org.elasticsearch.common.xcontent.XContentBuilder @@ -34,14 +33,11 @@ import java.time.Instant private val log = LogManager.getLogger(ISMTemplate::class.java) -// ComposableIndexTemplate -// ManagedIndexMetaData data class ISMTemplate( val indexPatterns: List<String>, - val policyID: String, val priority: Int, val lastUpdatedTime: Instant -) : ToXContentObject, AbstractDiffable<ISMTemplate>() { +) : ToXContentObject, Writeable { init { require(indexPatterns.isNotEmpty()) { "at least give one index pattern" } @@ -50,7 +46,6 @@ data class ISMTemplate( override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { return builder.startObject() .field(INDEX_PATTERN, indexPatterns) - .field(POLICY_ID, policyID) .field(PRIORITY, priority) .optionalTimeField(LAST_UPDATED_TIME_FIELD, lastUpdatedTime) .endObject() @@ -59,7 +54,6 @@ data class ISMTemplate( @Throws(IOException::class) constructor(sin: StreamInput) : this( sin.readStringList(), - sin.readString(), sin.readInt(), sin.readInstant() ) @@ -67,27 +61,23 @@ data class ISMTemplate( @Throws(IOException::class) override fun writeTo(out: StreamOutput) { out.writeStringCollection(indexPatterns) - out.writeString(policyID) out.writeInt(priority) out.writeInstant(lastUpdatedTime) } companion object { - const val ISM_TEMPLATE_ID = "template_name" const val ISM_TEMPLATE_TYPE = "ism_template" const val INDEX_PATTERN = "index_patterns" - const val POLICY_ID = "policy_id" const val PRIORITY = "priority" const val LAST_UPDATED_TIME_FIELD = "last_updated_time" @Suppress("ComplexMethod") fun parse(xcp: XContentParser): ISMTemplate { val indexPatterns: MutableList<String> = mutableListOf() - var policyID: String? = null var priority = 0 var lastUpdatedTime: Instant? = null - ensureExpectedToken(Token.START_OBJECT, xcp.nextToken(), xcp) + ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) while (xcp.nextToken() != Token.END_OBJECT) { val fieldName = xcp.currentName() xcp.nextToken() @@ -99,7 +89,6 @@ data class ISMTemplate( indexPatterns.add(xcp.text()) } } - POLICY_ID -> policyID = xcp.text() PRIORITY -> priority = if (xcp.currentToken() == Token.VALUE_NULL) 0 else xcp.intValue() LAST_UPDATED_TIME_FIELD -> lastUpdatedTime = xcp.instant() else -> throw IllegalArgumentException("Invalid field: [$fieldName] found in ISMTemplate.") @@ -107,13 +96,10 @@ data class ISMTemplate( } return ISMTemplate( - indexPatterns, - requireNotNull(policyID) { "policy id is null" }, - priority, - lastUpdatedTime ?: Instant.now() + indexPatterns = indexPatterns, + priority = priority, + lastUpdatedTime = lastUpdatedTime ?: Instant.now() ) } - - fun readISMTemplateDiffFrom(sin: StreamInput): Diff<ISMTemplate> = AbstractDiffable.readDiffFrom(::ISMTemplate, sin) } } diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateMetadata.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateMetadata.kt deleted file mode 100644 index bc39aae64..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateMetadata.kt +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model - -import org.apache.logging.log4j.LogManager -import org.elasticsearch.Version -import org.elasticsearch.cluster.Diff -import org.elasticsearch.cluster.DiffableUtils -import org.elasticsearch.cluster.NamedDiff -import org.elasticsearch.cluster.metadata.Metadata -import org.elasticsearch.common.ParseField -import org.elasticsearch.common.io.stream.StreamInput -import org.elasticsearch.common.io.stream.StreamOutput -import org.elasticsearch.common.xcontent.ToXContent -import org.elasticsearch.common.xcontent.XContentBuilder -import java.io.IOException -import java.util.EnumSet - -private val log = LogManager.getLogger(ISMTemplateMetadata::class.java) - -/** - * <template_name>: ISMTemplate - */ -// ComponentTemplateMetadata -// EnrichMetadata -class ISMTemplateMetadata(val ismTemplates: Map<String, ISMTemplate>) : Metadata.Custom { - - @Throws(IOException::class) - constructor(sin: StreamInput) : this( - sin.readMap(StreamInput::readString, ::ISMTemplate) - ) - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - out.writeMap(ismTemplates, StreamOutput::writeString) { stream, `val` -> `val`.writeTo(stream) } - } - - override fun diff(before: Metadata.Custom): Diff<Metadata.Custom> { - return ISMTemplateMetadataDiff((before as ISMTemplateMetadata), this) - } - - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - builder.startObject(ISM_TEMPLATE.preferredName) - ismTemplates.forEach { (k, v) -> - builder.field(k, v) - } - builder.endObject() - return builder - } - - override fun getWriteableName(): String = TYPE - - override fun getMinimalSupportedVersion(): Version = Version.V_7_7_0 - - override fun context(): EnumSet<Metadata.XContentContext> = Metadata.ALL_CONTEXTS - - class ISMTemplateMetadataDiff : NamedDiff<Metadata.Custom> { - - val ismTemplateDiff: Diff<Map<String, ISMTemplate>> - - constructor(before: ISMTemplateMetadata, after: ISMTemplateMetadata) { - this.ismTemplateDiff = DiffableUtils.diff(before.ismTemplates, after.ismTemplates, DiffableUtils.getStringKeySerializer()) - } - - @Throws(IOException::class) - constructor(sin: StreamInput) { - this.ismTemplateDiff = DiffableUtils.readJdkMapDiff(sin, DiffableUtils.getStringKeySerializer(), - ::ISMTemplate, ISMTemplate.Companion::readISMTemplateDiffFrom) - } - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - ismTemplateDiff.writeTo(out) - } - - override fun getWriteableName(): String = TYPE - - override fun apply(part: Metadata.Custom): Metadata.Custom { - return ISMTemplateMetadata(ismTemplateDiff.apply((part as ISMTemplateMetadata).ismTemplates)) - } - } - - companion object { - val TYPE = "ism_template" - val ISM_TEMPLATE = ParseField("ism_template") - - fun readDiffFrom(sin: StreamInput): NamedDiff<Metadata.Custom> = ISMTemplateMetadataDiff(sin) - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/Policy.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/Policy.kt index 69b2e79aa..e68261165 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/Policy.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/Policy.kt @@ -16,10 +16,10 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.instant +import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.optionalISMTemplateField import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.optionalTimeField import com.amazon.opendistroforelasticsearch.indexmanagement.util.IndexUtils import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.WITH_TYPE -import org.apache.logging.log4j.LogManager import org.elasticsearch.common.io.stream.StreamInput import org.elasticsearch.common.io.stream.StreamOutput import org.elasticsearch.common.io.stream.Writeable @@ -33,8 +33,6 @@ import org.elasticsearch.index.seqno.SequenceNumbers import java.io.IOException import java.time.Instant -private val log = LogManager.getLogger(Policy::class.java) - data class Policy( val id: String = NO_ID, val seqNo: Long = SequenceNumbers.UNASSIGNED_SEQ_NO, @@ -44,7 +42,8 @@ data class Policy( val lastUpdatedTime: Instant, val errorNotification: ErrorNotification?, val defaultState: String, - val states: List<State> + val states: List<State>, + val ismTemplate: ISMTemplate? = null ) : ToXContentObject, Writeable { init { @@ -75,6 +74,7 @@ data class Policy( .field(ERROR_NOTIFICATION_FIELD, errorNotification) .field(DEFAULT_STATE_FIELD, defaultState) .field(STATES_FIELD, states.toTypedArray()) + .optionalISMTemplateField(ISM_TEMPLATE, ismTemplate) if (params.paramAsBoolean(WITH_TYPE, true)) builder.endObject() return builder.endObject() } @@ -89,7 +89,8 @@ data class Policy( lastUpdatedTime = sin.readInstant(), errorNotification = sin.readOptionalWriteable(::ErrorNotification), defaultState = sin.readString(), - states = sin.readList(::State) + states = sin.readList(::State), + ismTemplate = sin.readOptionalWriteable(::ISMTemplate) ) @Throws(IOException::class) @@ -103,6 +104,7 @@ data class Policy( out.writeOptionalWriteable(errorNotification) out.writeString(defaultState) out.writeList(states) + out.writeOptionalWriteable(ismTemplate) } companion object { @@ -115,6 +117,7 @@ data class Policy( const val ERROR_NOTIFICATION_FIELD = "error_notification" const val DEFAULT_STATE_FIELD = "default_state" const val STATES_FIELD = "states" + const val ISM_TEMPLATE = "ism_template" @Suppress("ComplexMethod") @JvmStatic @@ -132,6 +135,7 @@ data class Policy( var lastUpdatedTime: Instant? = null var schemaVersion: Long = IndexUtils.DEFAULT_SCHEMA_VERSION val states: MutableList<State> = mutableListOf() + var ismTemplate: ISMTemplate? = null ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) while (xcp.nextToken() != Token.END_OBJECT) { @@ -151,6 +155,7 @@ data class Policy( states.add(State.parse(xcp)) } } + ISM_TEMPLATE -> ismTemplate = if (xcp.currentToken() == Token.VALUE_NULL) null else ISMTemplate.parse(xcp) else -> throw IllegalArgumentException("Invalid field: [$fieldName] found in Policy.") } } @@ -164,7 +169,8 @@ data class Policy( lastUpdatedTime ?: Instant.now(), errorNotification, requireNotNull(defaultState) { "$DEFAULT_STATE_FIELD is null" }, - states.toList() + states.toList(), + ismTemplate ) } } diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestAddISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestAddISMTemplateAction.kt deleted file mode 100644 index fe3774634..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestAddISMTemplateAction.kt +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler - -import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin.Companion.ISM_TEMPLATE_BASE_URI -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put.PutISMTemplateAction -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put.PutISMTemplateRequest -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put.PutISMTemplateResponse -import org.apache.logging.log4j.LogManager -import org.elasticsearch.action.support.master.MasterNodeRequest.DEFAULT_MASTER_NODE_TIMEOUT -import org.elasticsearch.client.node.NodeClient -import org.elasticsearch.common.xcontent.ToXContent -import org.elasticsearch.rest.BaseRestHandler -import org.elasticsearch.rest.BytesRestResponse -import org.elasticsearch.rest.RestHandler.Route -import org.elasticsearch.rest.RestRequest -import org.elasticsearch.rest.RestRequest.Method.PUT -import org.elasticsearch.rest.RestResponse -import org.elasticsearch.rest.RestStatus -import org.elasticsearch.rest.action.RestResponseListener -import java.lang.IllegalArgumentException -import java.time.Instant - -private val log = LogManager.getLogger(RestAddISMTemplateAction::class.java) - -class RestAddISMTemplateAction : BaseRestHandler() { - override fun routes(): List<Route> { - return listOf( - Route(PUT, ISM_TEMPLATE_BASE_URI), - Route(PUT, "$ISM_TEMPLATE_BASE_URI/{templateID}") - ) - } - - override fun getName(): String = "add_ism_template_action" - - override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { - log.info("${request.method()} $ISM_TEMPLATE_BASE_URI") - - val templateName = request.param("templateID", "") - if (templateName == "") { throw IllegalArgumentException("Missing template name") } - - val xcp = request.contentParser() - val ismTemplate = ISMTemplate.parse(xcp).copy(lastUpdatedTime = Instant.now()) - val masterTimeout = request.paramAsTime("master_timeout", DEFAULT_MASTER_NODE_TIMEOUT) - val addISMTemplateRequest = PutISMTemplateRequest(templateName, ismTemplate).masterNodeTimeout(masterTimeout) - - return RestChannelConsumer { channel -> - client.execute(PutISMTemplateAction.INSTANCE, addISMTemplateRequest, object : RestResponseListener<PutISMTemplateResponse>(channel) { - override fun buildResponse(response: PutISMTemplateResponse): RestResponse { - val restResponse = BytesRestResponse(response.status, response.toXContent(channel.newBuilder(), ToXContent.EMPTY_PARAMS)) - if (response.status == RestStatus.CREATED) { - val location = "$ISM_TEMPLATE_BASE_URI/${response.id}" - restResponse.addHeader("Location", location) - } - return restResponse - } - }) - } - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestDeleteISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestDeleteISMTemplateAction.kt deleted file mode 100644 index 74c709fe8..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestDeleteISMTemplateAction.kt +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler - -import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin.Companion.ISM_TEMPLATE_BASE_URI -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.delete.DeleteISMTemplateAction -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.delete.DeleteISMTemplateRequest -import org.apache.logging.log4j.LogManager -import org.elasticsearch.client.node.NodeClient -import org.elasticsearch.rest.BaseRestHandler -import org.elasticsearch.rest.RestHandler.Route -import org.elasticsearch.rest.RestRequest -import org.elasticsearch.rest.RestRequest.Method.DELETE -import org.elasticsearch.rest.action.RestToXContentListener -import java.lang.IllegalArgumentException - -private val log = LogManager.getLogger(RestDeleteISMTemplateAction::class.java) - -class RestDeleteISMTemplateAction : BaseRestHandler() { - override fun routes(): List<Route> { - return listOf( - Route(DELETE, "$ISM_TEMPLATE_BASE_URI/{templateID}") - ) - } - - override fun getName(): String = "remove_ism_template_action" - - override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { - log.info("${request.method()} $ISM_TEMPLATE_BASE_URI") - - val templateName = request.param("templateID", "") - if (templateName == "") { throw IllegalArgumentException("Missing template name") } - - val deleteISMTemplateRequest = DeleteISMTemplateRequest(templateName) - - return RestChannelConsumer { channel -> - client.execute(DeleteISMTemplateAction.INSTANCE, deleteISMTemplateRequest, RestToXContentListener(channel)) - } - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestGetISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestGetISMTemplateAction.kt deleted file mode 100644 index 0c8c2a05c..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/RestGetISMTemplateAction.kt +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler - -import org.apache.logging.log4j.LogManager -import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin.Companion.ISM_TEMPLATE_BASE_URI -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get.GetISMTemplateAction -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get.GetISMTemplateRequest -import org.elasticsearch.action.support.master.MasterNodeRequest.DEFAULT_MASTER_NODE_TIMEOUT -import org.elasticsearch.client.node.NodeClient -import org.elasticsearch.common.Strings -import org.elasticsearch.rest.BaseRestHandler -import org.elasticsearch.rest.RestHandler.Route -import org.elasticsearch.rest.RestRequest -import org.elasticsearch.rest.RestRequest.Method.GET -import org.elasticsearch.rest.action.RestToXContentListener - -private val log = LogManager.getLogger(RestGetISMTemplateAction::class.java) - -class RestGetISMTemplateAction : BaseRestHandler() { - override fun routes(): List<Route> { - return listOf( - Route(GET, ISM_TEMPLATE_BASE_URI), - Route(GET, "$ISM_TEMPLATE_BASE_URI/{templateID}") - ) - } - - override fun getName(): String = "get_ism_template_action" - - override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { - log.info("${request.method()} $ISM_TEMPLATE_BASE_URI") - - val templateNames = Strings.splitStringByCommaToArray(request.param("templateID")) - val masterTimeout = request.paramAsTime("master_timeout", DEFAULT_MASTER_NODE_TIMEOUT) - val getISMTemplateReq = GetISMTemplateRequest(templateNames).masterNodeTimeout(masterTimeout) - - return RestChannelConsumer { channel -> - client.execute(GetISMTemplateAction.INSTANCE, getISMTemplateReq, RestToXContentListener(channel)) - } - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/TransportIndexPolicyAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/TransportIndexPolicyAction.kt index 4a7f62e97..fc10a6ba6 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/TransportIndexPolicyAction.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/TransportIndexPolicyAction.kt @@ -17,6 +17,11 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanageme import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementIndices import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ISMTemplateService.Companion.findConflictingISMTemplates +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ISMTemplateService.Companion.validateFormat +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.ismTemplatesFromSearchResponse +import com.amazon.opendistroforelasticsearch.indexmanagement.util.IndexManagementException +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.ISM_TEMPLATE_FIELD import com.amazon.opendistroforelasticsearch.indexmanagement.util.IndexUtils import org.apache.logging.log4j.LogManager import org.elasticsearch.ElasticsearchStatusException @@ -24,16 +29,22 @@ import org.elasticsearch.action.ActionListener import org.elasticsearch.action.DocWriteRequest import org.elasticsearch.action.index.IndexRequest import org.elasticsearch.action.index.IndexResponse +import org.elasticsearch.action.search.SearchRequest +import org.elasticsearch.action.search.SearchResponse import org.elasticsearch.action.support.ActionFilters import org.elasticsearch.action.support.HandledTransportAction import org.elasticsearch.action.support.master.AcknowledgedResponse import org.elasticsearch.client.node.NodeClient import org.elasticsearch.common.inject.Inject +import org.elasticsearch.common.xcontent.NamedXContentRegistry import org.elasticsearch.common.xcontent.XContentFactory +import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.index.seqno.SequenceNumbers import org.elasticsearch.rest.RestStatus +import org.elasticsearch.search.builder.SearchSourceBuilder import org.elasticsearch.tasks.Task import org.elasticsearch.transport.TransportService +import java.util.stream.Collectors private val log = LogManager.getLogger(TransportIndexPolicyAction::class.java) @@ -41,7 +52,8 @@ class TransportIndexPolicyAction @Inject constructor( val client: NodeClient, transportService: TransportService, actionFilters: ActionFilters, - val ismIndices: IndexManagementIndices + val ismIndices: IndexManagementIndices, + val xContentRegistry: NamedXContentRegistry ) : HandledTransportAction<IndexPolicyRequest, IndexPolicyResponse>( IndexPolicyAction.NAME, transportService, actionFilters, ::IndexPolicyRequest ) { @@ -69,7 +81,12 @@ class TransportIndexPolicyAction @Inject constructor( private fun onCreateMappingsResponse(response: AcknowledgedResponse) { if (response.isAcknowledged) { log.info("Successfully created or updated ${IndexManagementPlugin.INDEX_MANAGEMENT_INDEX} with newest mappings.") - putPolicy() + + // if there is template field, we will check + val reqTemplate = request.policy.ismTemplate + if (reqTemplate != null) { + checkTemplate(reqTemplate.indexPatterns, reqTemplate.priority) + } else putPolicy() } else { log.error("Unable to create or update ${IndexManagementPlugin.INDEX_MANAGEMENT_INDEX} with newest mapping.") @@ -79,6 +96,41 @@ class TransportIndexPolicyAction @Inject constructor( } } + private fun checkTemplate(indexPatterns: List<String>, priority: Int) { + val possibleEx = validateFormat(indexPatterns) + if (possibleEx != null) { + actionListener.onFailure(possibleEx) + return + } + + val searchRequest = SearchRequest() + .source( + SearchSourceBuilder().query( + QueryBuilders.existsQuery(ISM_TEMPLATE_FIELD))) + .indices(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX) + + client.search(searchRequest, object : ActionListener<SearchResponse> { + override fun onResponse(response: SearchResponse) { + val ismTemplates = ismTemplatesFromSearchResponse(response, xContentRegistry) + val overlaps = findConflictingISMTemplates(request.policyID, indexPatterns, priority, ismTemplates) + if (overlaps.isNotEmpty()) { + val esg = "new policy ${request.policyID} has an ism template with index pattern $indexPatterns " + + "matching existing templates ${overlaps.entries.stream().map { "${it.key} => ${it.value}" }.collect( + Collectors.joining(","))}," + + " please use a different priority than $priority" + actionListener.onFailure(IndexManagementException.wrap(IllegalArgumentException(esg))) + return + } + + putPolicy() + } + + override fun onFailure(t: Exception) { + actionListener.onFailure(t) + } + }) + } + private fun putPolicy() { request.policy.copy(schemaVersion = IndexUtils.indexManagementConfigSchemaVersion) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateAction.kt deleted file mode 100644 index b90840132..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateAction.kt +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.delete - -import org.elasticsearch.action.ActionType -import org.elasticsearch.action.support.master.AcknowledgedResponse - -class DeleteISMTemplateAction : ActionType<AcknowledgedResponse>(NAME, ::AcknowledgedResponse) { - companion object { - val NAME = "cluster:admin/opendistro/ism/templates/remove" - val INSTANCE = DeleteISMTemplateAction() - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequest.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequest.kt deleted file mode 100644 index 4a767f862..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequest.kt +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.delete - -import org.elasticsearch.action.ActionRequestValidationException -import org.elasticsearch.action.support.master.MasterNodeRequest -import org.elasticsearch.common.io.stream.StreamInput -import org.elasticsearch.common.io.stream.StreamOutput -import java.io.IOException - -class DeleteISMTemplateRequest : MasterNodeRequest<DeleteISMTemplateRequest> { - - val templateName: String - - constructor( - templateName: String - ) : super() { - this.templateName = templateName - } - - @Throws(IOException::class) - constructor(sin: StreamInput) : super(sin) { - templateName = sin.readString() - } - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - super.writeTo(out) - out.writeString(templateName) - } - - override fun validate(): ActionRequestValidationException? { - return null - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/TransportDeleteISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/TransportDeleteISMTemplateAction.kt deleted file mode 100644 index 3fa4f2b02..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/TransportDeleteISMTemplateAction.kt +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.delete - -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ISMTemplateService -import org.apache.logging.log4j.LogManager -import org.elasticsearch.action.ActionListener -import org.elasticsearch.action.support.ActionFilters -import org.elasticsearch.action.support.master.AcknowledgedResponse -import org.elasticsearch.action.support.master.TransportMasterNodeAction -import org.elasticsearch.cluster.ClusterState -import org.elasticsearch.cluster.block.ClusterBlockException -import org.elasticsearch.cluster.block.ClusterBlockLevel -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver -import org.elasticsearch.cluster.service.ClusterService -import org.elasticsearch.common.inject.Inject -import org.elasticsearch.common.io.stream.StreamInput -import org.elasticsearch.common.io.stream.Writeable -import org.elasticsearch.threadpool.ThreadPool -import org.elasticsearch.transport.TransportService - -private val log = LogManager.getLogger(TransportDeleteISMTemplateAction::class.java) - -class TransportDeleteISMTemplateAction @Inject constructor( - transportService: TransportService, - clusterService: ClusterService, - threadPool: ThreadPool, - actionFilters: ActionFilters, - indexNameExpressionResolver: IndexNameExpressionResolver, - val ismTemplateService: ISMTemplateService -) : TransportMasterNodeAction<DeleteISMTemplateRequest, AcknowledgedResponse>( - DeleteISMTemplateAction.NAME, - transportService, - clusterService, - threadPool, - actionFilters, - Writeable.Reader { DeleteISMTemplateRequest(it) }, - indexNameExpressionResolver -) { - override fun executor(): String { - return ThreadPool.Names.SAME - } - - override fun read(sin: StreamInput): AcknowledgedResponse { - return AcknowledgedResponse(sin) - } - - override fun masterOperation(request: DeleteISMTemplateRequest, state: ClusterState, listener: ActionListener<AcknowledgedResponse>) { - ismTemplateService.deleteISMTemplate(request.templateName, request.masterNodeTimeout(), listener) - } - - override fun checkBlock(request: DeleteISMTemplateRequest, state: ClusterState): ClusterBlockException? { - return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE) - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateAction.kt deleted file mode 100644 index ebf148caa..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateAction.kt +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get - -import org.elasticsearch.action.ActionType - -class GetISMTemplateAction : ActionType<GetISMTemplateResponse>(NAME, ::GetISMTemplateResponse) { - companion object { - val NAME = "cluster:admin/opendistro/ism/templates/read" - val INSTANCE = GetISMTemplateAction() - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequest.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequest.kt deleted file mode 100644 index c3e7895ce..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequest.kt +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get - -import org.elasticsearch.action.ActionRequestValidationException -import org.elasticsearch.action.support.master.MasterNodeRequest -import org.elasticsearch.common.io.stream.StreamInput -import org.elasticsearch.common.io.stream.StreamOutput -import java.io.IOException - -class GetISMTemplateRequest : MasterNodeRequest<GetISMTemplateRequest> { - - // TODO not sure array is the right choice - val templateNames: Array<String> - - constructor(templateName: Array<String>) : super() { - this.templateNames = templateName - } - - @Throws(IOException::class) - constructor(sin: StreamInput) : super(sin) { - templateNames = sin.readStringArray() - } - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - super.writeTo(out) - out.writeStringArray(templateNames) - } - - override fun validate(): ActionRequestValidationException? { - return null - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponse.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponse.kt deleted file mode 100644 index 628e696b1..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponse.kt +++ /dev/null @@ -1,52 +0,0 @@ -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get - -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate -import org.elasticsearch.action.ActionResponse -import org.elasticsearch.common.io.stream.StreamInput -import org.elasticsearch.common.io.stream.StreamOutput -import org.elasticsearch.common.xcontent.ToXContent -import org.elasticsearch.common.xcontent.ToXContentObject -import org.elasticsearch.common.xcontent.XContentBuilder -import java.io.IOException - -// GetComposableIndexTemplateAction.Response -class GetISMTemplateResponse : ActionResponse, ToXContentObject { - - val ismTemplates: Map<String, ISMTemplate> - - constructor(ismTemplates: Map<String, ISMTemplate>) : super() { - this.ismTemplates = ismTemplates - } - - @Throws(IOException::class) - constructor(sin: StreamInput) : super(sin) { - val size = sin.readVInt() - ismTemplates = mutableMapOf() - repeat(size) { - ismTemplates.put(sin.readString(), ISMTemplate(sin)) - } - } - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - out.writeVInt(ismTemplates.size) - ismTemplates.forEach { (k, v) -> - out.writeString(k) - v.writeTo(out) - } - } - - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - builder.startObject().startArray(ISM_TEMPLATES) - ismTemplates.forEach { (k, v) -> - builder.startObject().field(TEMPLATE_NAME, k).field(ISM_TEMPLATE, v).endObject() - } - return builder.endArray().endObject() - } - - companion object { - val ISM_TEMPLATES = "ism_templates" - val TEMPLATE_NAME = "template_name" - val ISM_TEMPLATE = "ism_template" - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/TransportGetISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/TransportGetISMTemplateAction.kt deleted file mode 100644 index 7a811aefc..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/TransportGetISMTemplateAction.kt +++ /dev/null @@ -1,74 +0,0 @@ -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get - -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.ismTemplates -import org.apache.logging.log4j.LogManager -import org.elasticsearch.ResourceNotFoundException -import org.elasticsearch.action.ActionListener -import org.elasticsearch.action.support.ActionFilters -import org.elasticsearch.action.support.master.TransportMasterNodeAction -import org.elasticsearch.cluster.ClusterState -import org.elasticsearch.cluster.block.ClusterBlockException -import org.elasticsearch.cluster.block.ClusterBlockLevel -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver -import org.elasticsearch.cluster.service.ClusterService -import org.elasticsearch.common.inject.Inject -import org.elasticsearch.common.io.stream.StreamInput -import org.elasticsearch.common.io.stream.Writeable -import org.elasticsearch.common.regex.Regex -import org.elasticsearch.threadpool.ThreadPool -import org.elasticsearch.transport.TransportService - -private val log = LogManager.getLogger(TransportGetISMTemplateAction::class.java) - -// TransportGetComposableIndexTemplateAction -class TransportGetISMTemplateAction @Inject constructor( - transportService: TransportService, - clusterService: ClusterService, - threadPool: ThreadPool, - actionFilters: ActionFilters, - indexNameExpressionResolver: IndexNameExpressionResolver -) : TransportMasterNodeAction<GetISMTemplateRequest, GetISMTemplateResponse>( - GetISMTemplateAction.NAME, - transportService, - clusterService, - threadPool, - actionFilters, - Writeable.Reader { GetISMTemplateRequest(it) }, - indexNameExpressionResolver -) { - override fun executor(): String { - return ThreadPool.Names.SAME - } - - override fun read(sin: StreamInput): GetISMTemplateResponse { - return GetISMTemplateResponse(sin) - } - - override fun masterOperation(request: GetISMTemplateRequest, state: ClusterState, listener: ActionListener<GetISMTemplateResponse>) { - val allTemplates = state.metadata.ismTemplates() - if (request.templateNames.isEmpty()) { - listener.onResponse(GetISMTemplateResponse(allTemplates)) - return - } - - val results = mutableMapOf<String, ISMTemplate>() - val reqTemplates = request.templateNames - if (allTemplates.isEmpty()) throw ResourceNotFoundException("index template matching ${reqTemplates.toList()} not found") - reqTemplates.forEach { name -> - allTemplates.forEach { (templateName, template) -> - when { - Regex.simpleMatch(name, templateName) -> results[templateName] = template - allTemplates.containsKey(name) -> results[templateName] = template - else -> throw ResourceNotFoundException("index template matching [$name] not found") - } - } - } - - listener.onResponse(GetISMTemplateResponse(results)) - } - - override fun checkBlock(request: GetISMTemplateRequest, state: ClusterState): ClusterBlockException? { - return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE) - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateAction.kt deleted file mode 100644 index af65d2c14..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateAction.kt +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put - -import org.elasticsearch.action.ActionType - -class PutISMTemplateAction : ActionType<PutISMTemplateResponse>(NAME, ::PutISMTemplateResponse) { - companion object { - val NAME = "cluster:admin/opendistro/ism/templates/add" - val INSTANCE = PutISMTemplateAction() - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateRequest.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateRequest.kt deleted file mode 100644 index 64b6ea0ae..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateRequest.kt +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put - -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate -import org.elasticsearch.action.ActionRequestValidationException -import org.elasticsearch.action.support.master.MasterNodeRequest -import org.elasticsearch.common.io.stream.StreamInput -import org.elasticsearch.common.io.stream.StreamOutput -import java.io.IOException - -class PutISMTemplateRequest : MasterNodeRequest<PutISMTemplateRequest> { - - val templateName: String - val ismTemplate: ISMTemplate - - constructor( - templateName: String, - ismTemplate: ISMTemplate - ) : super() { - this.templateName = templateName - this.ismTemplate = ismTemplate - } - - @Throws(IOException::class) - constructor(sin: StreamInput) : super(sin) { - templateName = sin.readString() - ismTemplate = ISMTemplate(sin) - } - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - super.writeTo(out) - out.writeString(templateName) - ismTemplate.writeTo(out) - } - - override fun validate(): ActionRequestValidationException? { - return null - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponse.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponse.kt deleted file mode 100644 index e3fd71411..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponse.kt +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put - -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate.Companion.ISM_TEMPLATE_ID -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate.Companion.ISM_TEMPLATE_TYPE -import org.elasticsearch.action.ActionResponse -import org.elasticsearch.common.io.stream.StreamInput -import org.elasticsearch.common.io.stream.StreamOutput -import org.elasticsearch.common.xcontent.ToXContent -import org.elasticsearch.common.xcontent.ToXContentObject -import org.elasticsearch.common.xcontent.XContentBuilder -import org.elasticsearch.rest.RestStatus -import java.io.IOException - -class PutISMTemplateResponse : ActionResponse, ToXContentObject { - - val id: String - val template: ISMTemplate - val status: RestStatus - - constructor( - id: String, - template: ISMTemplate, - status: RestStatus - ) : super() { - this.id = id - this.template = template - this.status = status - } - - @Throws(IOException::class) - constructor(sin: StreamInput) : this( - id = sin.readString(), - template = ISMTemplate(sin), - status = sin.readEnum(RestStatus::class.java) - ) - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - out.writeString(id) - template.writeTo(out) - out.writeEnum(status) - } - - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - return builder.startObject() - .field(ISM_TEMPLATE_ID, id) - .field(ISM_TEMPLATE_TYPE, template) - .endObject() - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/TransportPutISMTemplateAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/TransportPutISMTemplateAction.kt deleted file mode 100644 index 066328fd1..000000000 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/TransportPutISMTemplateAction.kt +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put - -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ISMTemplateService -import org.apache.logging.log4j.LogManager -import org.elasticsearch.action.ActionListener -import org.elasticsearch.action.support.ActionFilters -import org.elasticsearch.action.support.master.TransportMasterNodeAction -import org.elasticsearch.cluster.ClusterState -import org.elasticsearch.cluster.block.ClusterBlockException -import org.elasticsearch.cluster.block.ClusterBlockLevel -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver -import org.elasticsearch.cluster.service.ClusterService -import org.elasticsearch.common.inject.Inject -import org.elasticsearch.common.io.stream.StreamInput -import org.elasticsearch.common.io.stream.Writeable -import org.elasticsearch.threadpool.ThreadPool -import org.elasticsearch.transport.TransportService - -private val log = LogManager.getLogger(TransportPutISMTemplateAction::class.java) - -class TransportPutISMTemplateAction @Inject constructor( - transportService: TransportService, - clusterService: ClusterService, - threadPool: ThreadPool, - actionFilters: ActionFilters, - indexNameExpressionResolver: IndexNameExpressionResolver, - val ismTemplateService: ISMTemplateService -) : TransportMasterNodeAction<PutISMTemplateRequest, PutISMTemplateResponse>( - PutISMTemplateAction.NAME, - transportService, - clusterService, - threadPool, - actionFilters, - Writeable.Reader { PutISMTemplateRequest(it) }, - indexNameExpressionResolver -) { - /** - * callbacks is inexpensive, this value may be - * {@link org.elasticsearch.threadpool.ThreadPool.Names#SAME SAME} (indicating that the callbacks will run on the same thread - * as the cluster state events are fired with) - */ - override fun executor(): String { - return ThreadPool.Names.SAME - } - - override fun read(sin: StreamInput): PutISMTemplateResponse { - return PutISMTemplateResponse(sin) - } - - override fun masterOperation(request: PutISMTemplateRequest, state: ClusterState, listener: ActionListener<PutISMTemplateResponse>) { - ismTemplateService.putISMTemplate(request.templateName, request.ismTemplate, request.masterNodeTimeout(), listener) - } - - override fun checkBlock(request: PutISMTemplateRequest, state: ClusterState): ClusterBlockException? { - return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE) - } -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt index 455629aaf..b55b61cfd 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt @@ -18,10 +18,7 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanageme import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.optionalTimeField import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ChangePolicy -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplateMetadata import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig -import org.elasticsearch.cluster.metadata.Metadata import org.elasticsearch.common.io.stream.StreamInput import org.elasticsearch.common.io.stream.StreamOutput import org.elasticsearch.common.io.stream.Writeable @@ -38,6 +35,8 @@ const val FAILURES = "failures" const val FAILED_INDICES = "failed_indices" const val UPDATED_INDICES = "updated_indices" +const val ISM_TEMPLATE_FIELD = "policy.ism_template" + fun buildInvalidIndexResponse(builder: XContentBuilder, failedIndices: List<FailedIndex>) { if (failedIndices.isNotEmpty()) { builder.field(FAILURES, true) @@ -95,21 +94,3 @@ fun getPartialChangePolicyBuilder( .endObject() .endObject() } - -/** - * return sorted ism templates map saved in cluster metadata - */ -fun Metadata.ismTemplates(): Map<String, ISMTemplate> { - val ismCustomMetadata: ISMTemplateMetadata? = this.custom(ISMTemplateMetadata.TYPE) - return ismCustomMetadata?.ismTemplates?.toSortedMap() ?: emptyMap() -} - -fun Metadata.Builder.putISMTemplate(name: String, template: ISMTemplate, existing: Map<String, ISMTemplate>): Metadata.Builder { - return this.putCustom(ISMTemplateMetadata.TYPE, - ISMTemplateMetadata(existing.plus(name to template))) -} - -fun Metadata.Builder.removeISMTemplate(name: String, existing: Map<String, ISMTemplate>): Metadata.Builder { - return this.putCustom(ISMTemplateMetadata.TYPE, - ISMTemplateMetadata(existing.minus(name))) -} diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/util/IndexManagementException.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/util/IndexManagementException.kt new file mode 100644 index 000000000..4f8200db7 --- /dev/null +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/util/IndexManagementException.kt @@ -0,0 +1,45 @@ +package com.amazon.opendistroforelasticsearch.indexmanagement.util + +import org.elasticsearch.ElasticsearchException +import org.elasticsearch.common.Strings +import org.elasticsearch.common.ValidationException +import org.elasticsearch.index.IndexNotFoundException +import org.elasticsearch.rest.RestStatus +import java.lang.IllegalArgumentException + +class IndexManagementException(message: String, val status: RestStatus, ex: Exception) : ElasticsearchException(message, ex) { + + override fun status(): RestStatus { + return status + } + + companion object { + @JvmStatic + fun wrap(ex: Exception): ElasticsearchException { + + var friendlyMsg = ex.message as String + var status = RestStatus.INTERNAL_SERVER_ERROR + when (ex) { + is IndexNotFoundException -> { + status = ex.status() + friendlyMsg = "Configuration index not found" + } + is IllegalArgumentException -> { + status = RestStatus.BAD_REQUEST + friendlyMsg = ex.message as String + } + is ValidationException -> { + status = RestStatus.BAD_REQUEST + friendlyMsg = ex.message as String + } + else -> { + if (!Strings.isNullOrEmpty(ex.message)) { + friendlyMsg = ex.message as String + } + } + } + + return IndexManagementException(friendlyMsg, status, Exception("${ex.javaClass.name}: ${ex.message}")) + } + } +} diff --git a/src/main/resources/mappings/opendistro-ism-config.json b/src/main/resources/mappings/opendistro-ism-config.json index e98c644ef..02365218b 100644 --- a/src/main/resources/mappings/opendistro-ism-config.json +++ b/src/main/resources/mappings/opendistro-ism-config.json @@ -442,6 +442,23 @@ } } } + }, + "ism_template": { + "properties": { + "template_name": { + "type": "keyword" + }, + "index_patterns": { + "type": "keyword" + }, + "priority": { + "type": "long" + }, + "last_updated_time": { + "type": "date", + "format": "strict_date_time||epoch_millis" + } + } } } }, diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt index 1b5a44ec5..a5e2061a1 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt @@ -21,7 +21,6 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlug import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementIndices import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin.Companion.ISM_BASE_URI -import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin.Companion.ISM_TEMPLATE_BASE_URI import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementRestTestCase import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.parseWithType import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ChangePolicy @@ -36,9 +35,6 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagemen import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StateMetaData import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler.RestExplainAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get.GetISMTemplateResponse.Companion.ISM_TEMPLATE -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get.GetISMTemplateResponse.Companion.ISM_TEMPLATES -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get.GetISMTemplateResponse.Companion.TEMPLATE_NAME import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.FAILED_INDICES import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.FAILURES import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.UPDATED_INDICES @@ -703,76 +699,6 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() return true } - protected fun createISMTemplate( - name: String, - template: ISMTemplate - ): Response { - return createISMTemplateJson(name, template.toJsonString()) - } - - protected fun createISMTemplateJson( - name: String, - templateString: String - ): Response { - return client().makeRequest( - "PUT", - "$ISM_TEMPLATE_BASE_URI/$name", - StringEntity(templateString, APPLICATION_JSON) - ) - } - - protected fun getISMTemplatesAsMap(name: String): Map<String, Any> { - val response = client().makeRequest("GET", "$ISM_TEMPLATE_BASE_URI/$name") - assertEquals("Unexpected RestStatus", RestStatus.OK, response.restStatus()) - return response.asMap() - } - - protected fun getISMTemplatesAsObject(name: String?): Map<String, ISMTemplate> { - var endpoint = ISM_TEMPLATE_BASE_URI - if (name != null) endpoint += "/$name" - val response = client().makeRequest("GET", endpoint) - assertEquals("Unable to get template $name", RestStatus.OK, response.restStatus()) - - val xcp = createParser(XContentType.JSON.xContent(), response.entity.content) - val ismTemplates = mutableMapOf<String, ISMTemplate>() - - ensureExpectedToken(Token.START_OBJECT, xcp.nextToken(), xcp) - - while (xcp.nextToken() != Token.END_OBJECT) { - val fieldName = xcp.currentName() - xcp.nextToken() - - when (fieldName) { - ISM_TEMPLATES -> { - ensureExpectedToken(Token.START_ARRAY, xcp.currentToken(), xcp) - - var templateName: String? = null - var template: ISMTemplate? = null - - while (xcp.nextToken() != Token.END_ARRAY) { - when (xcp.currentName()) { - TEMPLATE_NAME -> { - xcp.nextToken() - templateName = xcp.text() - } - ISM_TEMPLATE -> { - // xcp.nextToken() - template = ISMTemplate.parse(xcp) - } - } - if (templateName != null && template != null) { - ismTemplates[templateName] = template - templateName = null - template = null - } - } - } - } - } - - return ismTemplates - } - protected fun assertPredicatesOnISMTemplatesMap( templatePredicates: List<Pair<String, List<Pair<String, (Any?) -> Boolean>>>>, // response map name: predicate response: Map<String, Any?> @@ -792,7 +718,6 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() protected fun assertISMTemplateEquals(expected: ISMTemplate, actualISMTemplateMap: Any?): Boolean { actualISMTemplateMap as Map<String, Any> assertEquals(expected.indexPatterns, actualISMTemplateMap[ISMTemplate.INDEX_PATTERN]) - assertEquals(expected.policyID, actualISMTemplateMap[ISMTemplate.POLICY_ID]) assertEquals(expected.priority, actualISMTemplateMap[ISMTemplate.PRIORITY]) return true } @@ -801,13 +726,8 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() assertNotNull(actual) if (actual != null) { assertEquals(expected.indexPatterns, actual.indexPatterns) - assertEquals(expected.policyID, actual.policyID) assertEquals(expected.priority, actual.priority) } return true } - - protected fun deleteISMTemplate(name: String): Response { - return client().makeRequest("DELETE", "$ISM_TEMPLATE_BASE_URI/$name") - } } diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/TestHelpers.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/TestHelpers.kt index 37e3557c6..67097dc71 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/TestHelpers.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/TestHelpers.kt @@ -67,10 +67,11 @@ fun randomPolicy( schemaVersion: Long = ESRestTestCase.randomLong(), lastUpdatedTime: Instant = Instant.now().truncatedTo(ChronoUnit.MILLIS), errorNotification: ErrorNotification? = randomErrorNotification(), - states: List<State> = List(ESRestTestCase.randomIntBetween(1, 10)) { randomState() } + states: List<State> = List(ESRestTestCase.randomIntBetween(1, 10)) { randomState() }, + ismTemplate: ISMTemplate? = null ): Policy { return Policy(id = id, schemaVersion = schemaVersion, lastUpdatedTime = lastUpdatedTime, - errorNotification = errorNotification, defaultState = states[0].name, states = states, description = description) + errorNotification = errorNotification, defaultState = states[0].name, states = states, description = description, ismTemplate = ismTemplate) } fun randomState( @@ -317,13 +318,11 @@ fun randomSweptManagedIndexConfig( fun randomISMTemplate( indexPatterns: List<String> = listOf(ESRestTestCase.randomAlphaOfLength(10) + "*"), - policyID: String = ESRestTestCase.randomAlphaOfLength(10), - priority: Int = ESRestTestCase.randomIntBetween(0, 200), + priority: Int = ESRestTestCase.randomIntBetween(0, 100), lastUpdatedTime: Instant = Instant.now().truncatedTo(ChronoUnit.MILLIS) ): ISMTemplate { return ISMTemplate( indexPatterns = indexPatterns, - policyID = policyID, priority = priority, lastUpdatedTime = lastUpdatedTime ) diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt index 1c2287a7c..5b6f39cde 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt @@ -1,21 +1,18 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler -import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin.Companion.ISM_TEMPLATE_BASE_URI import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.Policy import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.State import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.action.ReadOnlyActionConfig import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.randomErrorNotification +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.randomPolicy import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings -import com.amazon.opendistroforelasticsearch.indexmanagement.makeRequest import com.amazon.opendistroforelasticsearch.indexmanagement.randomInstant import com.amazon.opendistroforelasticsearch.indexmanagement.waitFor import org.elasticsearch.client.ResponseException import org.elasticsearch.common.settings.Settings import org.elasticsearch.rest.RestStatus -import org.elasticsearch.rest.RestRequest.Method.GET -import org.junit.After import java.time.Instant import java.time.temporal.ChronoUnit import java.util.Locale @@ -24,104 +21,33 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { private val testIndexName = javaClass.simpleName.toLowerCase(Locale.ROOT) - private val templateName = "t1" - private val templateName2 = "t2" - - @After - fun `clean template`() { - deleteISMTemplate(templateName) - deleteISMTemplate(templateName2) - } - - fun `test ISM template`() { - val ismTemp = ISMTemplate(listOf("log*"), "policy_1", 100, randomInstant()) - - var res = createISMTemplate(templateName, ismTemp) - assertEquals("Unable to create new ISM template", RestStatus.CREATED, res.restStatus()) - - res = createISMTemplate(templateName, ismTemp) - assertEquals("Unable to update new ISM template", RestStatus.OK, res.restStatus()) - - var getRes = getISMTemplatesAsObject(templateName) - assertISMTemplateEquals(ismTemp, getRes[templateName]) - - val ismTemp2 = ISMTemplate(listOf("trace*"), "policy_1", 100, randomInstant()) - createISMTemplate(templateName2, ismTemp2) - getRes = getISMTemplatesAsObject("$templateName,$templateName2") - val getRes2 = getISMTemplatesAsObject(null) - assertEquals(getRes, getRes2) - assertISMTemplateEquals(ismTemp, getRes[templateName]) - assertISMTemplateEquals(ismTemp2, getRes[templateName2]) - - val delRes = deleteISMTemplate(templateName) - assertEquals(true, delRes.asMap()["acknowledged"]) - } - - fun `test get not exist template`() { - try { - client().makeRequest(GET.toString(), "$ISM_TEMPLATE_BASE_URI/$templateName") - fail("Expect a failure") - } catch (e: ResponseException) { - assertEquals("Unexpected RestStatus", RestStatus.NOT_FOUND, e.response.restStatus()) - val actualMessage = e.response.asMap() - val expectErrorMessage = mapOf( - "error" to mapOf( - "root_cause" to listOf<Map<String, Any>>( - mapOf("type" to "resource_not_found_exception", "reason" to "index template matching [$templateName] not found") - ), - "type" to "resource_not_found_exception", - "reason" to "index template matching [$templateName] not found" - ), - "status" to 404 - ) - assertEquals(expectErrorMessage, actualMessage) - } - } + private val policyID1 = "t1" + private val policyID2 = "t2" fun `test add template with invalid index pattern`() { try { - val ismTemp = ISMTemplate(listOf(" "), "policy_1", 100, randomInstant()) - createISMTemplate(templateName, ismTemp) + val ismTemp = ISMTemplate(listOf(" "), 100, randomInstant()) + createPolicy(randomPolicy(ismTemplate = ismTemp), policyID1) fail("Expect a failure") } catch (e: ResponseException) { assertEquals("Unexpected RestStatus", RestStatus.BAD_REQUEST, e.response.restStatus()) - val actualMessage = e.response.asMap() - val expectErrorMessage = mapOf( - "error" to mapOf( - "reason" to "index_template [$templateName] invalid, cause [Validation Failed: 1: index_patterns [ ] must not contain a space;2: index_pattern [ ] must not contain the following characters [ , \", *, \\, <, |, ,, >, /, ?];]", - "type" to "invalid_index_template_exception", - "root_cause" to listOf<Map<String, Any>>( - mapOf("reason" to "index_template [$templateName] invalid, cause [Validation Failed: 1: index_patterns [ ] must not contain a space;2: index_pattern [ ] must not contain the following characters [ , \", *, \\, <, |, ,, >, /, ?];]", - "type" to "invalid_index_template_exception") - ) - ), - "status" to 400 - ) - assertEquals(expectErrorMessage, actualMessage) + val actualMessage = e.response.asMap()["error"] as Map<String, Any> + val expectedReason = "Validation Failed: 1: index_patterns [ ] must not contain a space;2: index_pattern [ ] must not contain the following characters [ , \", *, \\, <, |, ,, >, /, ?];" + assertEquals(expectedReason, actualMessage["reason"]) } } fun `test add template with overlapping index pattern`() { try { - val ismTemp = ISMTemplate(listOf("log*"), "policy_1", 100, randomInstant()) - createISMTemplate(templateName, ismTemp) - createISMTemplate(templateName2, ismTemp) + val ismTemp = ISMTemplate(listOf("log*"), 100, randomInstant()) + createPolicy(randomPolicy(ismTemplate = ismTemp), policyID1) + createPolicy(randomPolicy(ismTemplate = ismTemp), policyID2) fail("Expect a failure") } catch (e: ResponseException) { assertEquals("Unexpected RestStatus", RestStatus.BAD_REQUEST, e.response.restStatus()) - val actualMessage = e.response.asMap() - val expectErrorMessage = mapOf( - "error" to mapOf( - "reason" to "new ism template $templateName2 has index pattern [log*] matching existing templates t1 => [log*], please use a different priority than 100", - "type" to "illegal_argument_exception", - "root_cause" to listOf<Map<String, Any>>( - mapOf("reason" to "new ism template $templateName2 has index pattern [log*] matching existing templates t1 => [log*], please use a different priority than 100", - "type" to "illegal_argument_exception") - ) - ), - "status" to 400 - ) - assertEquals(expectErrorMessage, actualMessage) + val actualMessage = e.response.asMap()["error"] as Map<String, Any> + val expectedReason = "new policy $policyID2 has an ism template with index pattern [log*] matching existing templates $policyID1 => [log*], please use a different priority than 100" + assertEquals(expectedReason, actualMessage["reason"]) } } @@ -134,8 +60,7 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { // need to specify policyID null, can remove after policyID deprecated createIndex(indexName1, null) - val ismTemp = ISMTemplate(listOf("log*"), policyID, 100, randomInstant()) - createISMTemplate(templateName, ismTemp) + val ismTemp = ISMTemplate(listOf("log*"), 100, randomInstant()) val actionConfig = ReadOnlyActionConfig(0) val states = listOf( @@ -148,7 +73,8 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { lastUpdatedTime = Instant.now().truncatedTo(ChronoUnit.MILLIS), errorNotification = randomErrorNotification(), defaultState = states[0].name, - states = states + states = states, + ismTemplate = ismTemp ) createPolicy(policy, policyID) diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ActionTests.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ActionTests.kt index f5a9459e9..003329b8f 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ActionTests.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ActionTests.kt @@ -21,9 +21,6 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagemen import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.indexpolicy.IndexPolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.explain.ExplainAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.getpolicy.GetPolicyAction -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.delete.DeleteISMTemplateAction -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get.GetISMTemplateAction -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put.PutISMTemplateAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.removepolicy.RemovePolicyAction import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.retryfailedmanagedindex.RetryFailedManagedIndexAction import org.elasticsearch.test.ESTestCase @@ -68,19 +65,4 @@ class ActionTests : ESTestCase() { assertNotNull(GetPolicyAction.NAME) assertEquals(GetPolicyAction.INSTANCE.name(), GetPolicyAction.NAME) } - - fun `test get template action name`() { - assertNotNull(GetISMTemplateAction.NAME) - assertEquals(GetISMTemplateAction.INSTANCE.name(), GetISMTemplateAction.NAME) - } - - fun `test put template action name`() { - assertNotNull(PutISMTemplateAction.NAME) - assertEquals(PutISMTemplateAction.INSTANCE.name(), PutISMTemplateAction.NAME) - } - - fun `test delete template action name`() { - assertNotNull(DeleteISMTemplateAction.NAME) - assertEquals(DeleteISMTemplateAction.INSTANCE.name(), DeleteISMTemplateAction.NAME) - } } diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequestTests.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequestTests.kt deleted file mode 100644 index 260cae0ee..000000000 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/delete/DeleteISMTemplateRequestTests.kt +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.delete - -import org.elasticsearch.common.io.stream.BytesStreamOutput -import org.elasticsearch.common.io.stream.StreamInput -import org.elasticsearch.test.ESTestCase - -class DeleteISMTemplateRequestTests : ESTestCase() { - - fun `test delete template request`() { - val templateName = "t1" - val req = DeleteISMTemplateRequest(templateName) - - val out = BytesStreamOutput() - req.writeTo(out) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newReq = DeleteISMTemplateRequest(sin) - assertEquals(templateName, newReq.templateName) - } -} diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequestTests.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequestTests.kt deleted file mode 100644 index 68a8e3721..000000000 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateRequestTests.kt +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get - -import org.elasticsearch.common.io.stream.BytesStreamOutput -import org.elasticsearch.common.io.stream.StreamInput -import org.elasticsearch.test.ESTestCase - -class GetISMTemplateRequestTests : ESTestCase() { - - fun `test get templates request`() { - val templateNames = arrayOf("t1") - val req = GetISMTemplateRequest(templateNames) - - val out = BytesStreamOutput() - req.writeTo(out) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newReq = GetISMTemplateRequest(sin) - assertEquals(templateNames, newReq.templateNames) - } -} diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponseTests.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponseTests.kt deleted file mode 100644 index e55ad8ec8..000000000 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/get/GetISMTemplateResponseTests.kt +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.get - -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate -import com.amazon.opendistroforelasticsearch.indexmanagement.randomInstant -import org.elasticsearch.common.io.stream.BytesStreamOutput -import org.elasticsearch.common.io.stream.StreamInput -import org.elasticsearch.test.ESTestCase - -class GetISMTemplateResponseTests : ESTestCase() { - - fun `test get templates response`() { - val ismTemplates = mapOf("t1" to ISMTemplate(listOf("log*"), "policy_1", 100, randomInstant())) - val req = GetISMTemplateResponse(ismTemplates) - - val out = BytesStreamOutput() - req.writeTo(out) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newReq = GetISMTemplateResponse(sin) - assertEquals(ismTemplates, newReq.ismTemplates) - } -} diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateRequestTests.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateRequestTests.kt deleted file mode 100644 index b8747eb4e..000000000 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateRequestTests.kt +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put - -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate -import com.amazon.opendistroforelasticsearch.indexmanagement.randomInstant -import org.elasticsearch.common.io.stream.BytesStreamOutput -import org.elasticsearch.common.io.stream.StreamInput -import org.elasticsearch.test.ESTestCase - -class PutISMTemplateRequestTests : ESTestCase() { - - fun `test put template request`() { - val templateName = "t1" - val ismTemplate = ISMTemplate(listOf("log*"), "policy_1", 100, randomInstant()) - val req = PutISMTemplateRequest(templateName, ismTemplate) - - val out = BytesStreamOutput() - req.writeTo(out) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newReq = PutISMTemplateRequest(sin) - assertEquals(templateName, newReq.templateName) - assertEquals(ismTemplate, newReq.ismTemplate) - } -} \ No newline at end of file diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponseTests.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponseTests.kt deleted file mode 100644 index 3d9f25c97..000000000 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/ismtemplate/put/PutISMTemplateResponseTests.kt +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.transport.action.ismtemplate.put - -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate -import com.amazon.opendistroforelasticsearch.indexmanagement.randomInstant -import org.elasticsearch.common.io.stream.BytesStreamOutput -import org.elasticsearch.common.io.stream.StreamInput -import org.elasticsearch.rest.RestStatus -import org.elasticsearch.test.ESTestCase - -class PutISMTemplateResponseTests : ESTestCase() { - - fun `test put template response`() { - val id = "t1" - val template = ISMTemplate(listOf("log*"), "policy_1", 100, randomInstant()) - val status = RestStatus.OK - val req = PutISMTemplateResponse(id, template, status) - - val out = BytesStreamOutput() - req.writeTo(out) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newReq = PutISMTemplateResponse(sin) - assertEquals(id, newReq.id) - assertEquals(template, newReq.template) - assertEquals(status, newReq.status) - } -} diff --git a/src/test/resources/mappings/cached-opendistro-ism-config.json b/src/test/resources/mappings/cached-opendistro-ism-config.json index e98c644ef..02365218b 100644 --- a/src/test/resources/mappings/cached-opendistro-ism-config.json +++ b/src/test/resources/mappings/cached-opendistro-ism-config.json @@ -442,6 +442,23 @@ } } } + }, + "ism_template": { + "properties": { + "template_name": { + "type": "keyword" + }, + "index_patterns": { + "type": "keyword" + }, + "priority": { + "type": "long" + }, + "last_updated_time": { + "type": "date", + "format": "strict_date_time||epoch_millis" + } + } } } }, From b69d2244e55dacbf7ddf64a242853c44e6105280 Mon Sep 17 00:00:00 2001 From: bowenlan-amzn <bowenlan@amazon.com> Date: Thu, 21 Jan 2021 16:48:00 -0800 Subject: [PATCH 16/21] address Ravi comments --- .../ISMTemplateService.kt | 29 +++++++++---------- .../ManagedIndexCoordinator.kt | 20 ++++++------- .../elasticapi/ElasticExtensions.kt | 2 +- .../indexpolicy/TransportIndexPolicyAction.kt | 10 +++---- .../resthandler/ISMTemplateRestAPIIT.kt | 2 +- 5 files changed, 31 insertions(+), 32 deletions(-) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt index a8d5849f7..eafd23e7c 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt @@ -31,7 +31,7 @@ private val log = LogManager.getLogger(ISMTemplateService::class.java) class ISMTemplateService { companion object { /** - * find the matching template for the index + * find the matching policy based on ISM template field for the given index * * filter out hidden index * filter out older index than template lastUpdateTime @@ -41,32 +41,31 @@ class ISMTemplateService { * @return policyID */ @Suppress("ReturnCount") - fun findMatchingISMTemplate(ismTemplates: Map<String, ISMTemplate>, indexMetadata: IndexMetadata): String? { + fun findMatchingPolicy(ismTemplates: Map<String, ISMTemplate>, indexMetadata: IndexMetadata): String? { + if (ismTemplates.isEmpty()) return null + val indexName = indexMetadata.index.name // don't include hidden index val isHidden = IndexMetadata.INDEX_HIDDEN_SETTING.get(indexMetadata.settings) - log.info("index $indexName is hidden $isHidden") if (isHidden) return null // only process indices created after template // traverse all ism templates for matching ones val patternMatchPredicate = { pattern: String -> Regex.simpleMatch(pattern, indexName) } - val matchedTemplates = mutableMapOf<ISMTemplate, String>() + var matchedPolicy: String? = null + var highestPriority: Int = -1 ismTemplates.filter { (_, template) -> - log.info("index create after template? ${template.lastUpdatedTime.toEpochMilli() < indexMetadata.creationDate}") template.lastUpdatedTime.toEpochMilli() < indexMetadata.creationDate - }.forEach { (templateName, template) -> + }.forEach { (policyID, template) -> val matched = template.indexPatterns.stream().anyMatch(patternMatchPredicate) - if (matched) matchedTemplates[template] = templateName + if (matched && highestPriority < template.priority) { + highestPriority = template.priority + matchedPolicy = policyID + } } - if (matchedTemplates.isEmpty()) return null - - // sort by template priority - val winner = matchedTemplates.keys.maxBy { it.priority } - log.info("winner with highest priority is $winner") - return matchedTemplates[winner] + return matchedPolicy } /** @@ -107,12 +106,12 @@ class ISMTemplateService { } /** - * find templates whose index patterns overlap with given template + * find policy templates whose index patterns overlap with given template * * @return map of overlapping template name to its index patterns */ @Suppress("SpreadOperator") - fun findConflictingISMTemplates( + fun findConflictingPolicyTemplates( candidate: String, indexPatterns: List<String>, priority: Int, diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt index bd621050b..eea930039 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt @@ -19,14 +19,14 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlug import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementIndices import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.parseWithType -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ISMTemplateService.Companion.findMatchingISMTemplate +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ISMTemplateService.Companion.findMatchingPolicy import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.getClusterStateManagedIndexConfig import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.getManagedIndexMetaData import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.getPolicyID import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.retry import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.suspendUntil import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.filterNotNullValues -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.ismTemplatesFromSearchResponse +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.getPolicyToTemplateMap import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.shouldCreateManagedIndexConfig import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.shouldDeleteManagedIndexConfig import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.shouldDeleteManagedIndexMetaData @@ -290,22 +290,22 @@ class ManagedIndexCoordinator( } // check if newly created indices matching any ISM templates - val updateMatchingIndexReqs = getMatchingIndicesUpdateReqs(event.state(), event.indicesCreated()) - if (updateMatchingIndexReqs.isNotEmpty()) hasCreateRequests = true + val updateMatchingIndexReq = getMatchingIndicesUpdateReq(event.state(), event.indicesCreated()) + if (updateMatchingIndexReq.isNotEmpty()) hasCreateRequests = true - updateManagedIndices(updateManagedIndicesRequests + updateMatchingIndexReqs + indicesDeletedRequests, hasCreateRequests) + updateManagedIndices(updateManagedIndicesRequests + updateMatchingIndexReq + indicesDeletedRequests, hasCreateRequests) clearManagedIndexMetaData(indicesToRemoveManagedIndexMetaDataFrom) } /** * build requests to create jobs for indices matching ISM templates */ - suspend fun getMatchingIndicesUpdateReqs(clusterState: ClusterState, indexNames: List<String>): List<DocWriteRequest<*>> { + suspend fun getMatchingIndicesUpdateReq(clusterState: ClusterState, indexNames: List<String>): List<DocWriteRequest<*>> { val indexMetadatas = clusterState.metadata.indices val templates = getISMTemplates() val indexToMatchedPolicy = indexNames.map { indexName -> - indexName to findMatchingISMTemplate(templates, indexMetadatas[indexName]) + indexName to findMatchingPolicy(templates, indexMetadatas[indexName]) }.toMap() val updateManagedIndexReqs = mutableListOf<DocWriteRequest<*>>() @@ -313,7 +313,7 @@ class ManagedIndexCoordinator( .forEach { (index, policyID) -> val indexUuid = indexMetadatas[index].indexUUID if (indexUuid != null) { - logger.info("auto manage index $index to policy $policyID") + logger.info("index [$index] will be managed by policy [$policyID]") updateManagedIndexReqs.add( managedIndexConfigIndexRequest(index, indexUuid, policyID, jobInterval)) } @@ -331,7 +331,7 @@ class ManagedIndexCoordinator( return try { val response: SearchResponse = client.suspendUntil { search(searchRequest, it) } - ismTemplatesFromSearchResponse(response).filterNotNullValues() + getPolicyToTemplateMap(response).filterNotNullValues() } catch (ex: IndexNotFoundException) { emptyMap() } catch (ex: ClusterBlockException) { @@ -395,7 +395,7 @@ class ManagedIndexCoordinator( // check all un-managed indices, if matches any ism template val unManagedIndices = clusterService.state().metadata.indices.values().filterNotNull() .filter { it.value.indexUUID !in currentManagedIndices.keys }.map { it.value.index.name } - val updateMatchingIndicesReqs = getMatchingIndicesUpdateReqs(clusterService.state(), unManagedIndices) + val updateMatchingIndicesReqs = getMatchingIndicesUpdateReq(clusterService.state(), unManagedIndices) updateManagedIndices(updateMatchingIndicesReqs, updateMatchingIndicesReqs.isNotEmpty()) val clusterStateManagedIndices = sweepClusterState(clusterService.state()) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/elasticapi/ElasticExtensions.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/elasticapi/ElasticExtensions.kt index e7bc7ca69..060eb0457 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/elasticapi/ElasticExtensions.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/elasticapi/ElasticExtensions.kt @@ -113,7 +113,7 @@ fun IndexMetadata.getManagedIndexMetaData(): ManagedIndexMetaData? { * * @return map of policyID to ISMTemplate in this policy */ -fun ismTemplatesFromSearchResponse(response: SearchResponse, xContentRegistry: NamedXContentRegistry = NamedXContentRegistry.EMPTY): +fun getPolicyToTemplateMap(response: SearchResponse, xContentRegistry: NamedXContentRegistry = NamedXContentRegistry.EMPTY): Map<String, ISMTemplate?> { return response.hits.hits.map { val id = it.id diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/TransportIndexPolicyAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/TransportIndexPolicyAction.kt index fc10a6ba6..a32d4938a 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/TransportIndexPolicyAction.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/TransportIndexPolicyAction.kt @@ -17,9 +17,9 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanageme import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementIndices import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ISMTemplateService.Companion.findConflictingISMTemplates +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ISMTemplateService.Companion.findConflictingPolicyTemplates import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ISMTemplateService.Companion.validateFormat -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.ismTemplatesFromSearchResponse +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.getPolicyToTemplateMap import com.amazon.opendistroforelasticsearch.indexmanagement.util.IndexManagementException import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.ISM_TEMPLATE_FIELD import com.amazon.opendistroforelasticsearch.indexmanagement.util.IndexUtils @@ -111,11 +111,11 @@ class TransportIndexPolicyAction @Inject constructor( client.search(searchRequest, object : ActionListener<SearchResponse> { override fun onResponse(response: SearchResponse) { - val ismTemplates = ismTemplatesFromSearchResponse(response, xContentRegistry) - val overlaps = findConflictingISMTemplates(request.policyID, indexPatterns, priority, ismTemplates) + val policyToTemplateMap = getPolicyToTemplateMap(response, xContentRegistry) + val overlaps = findConflictingPolicyTemplates(request.policyID, indexPatterns, priority, policyToTemplateMap) if (overlaps.isNotEmpty()) { val esg = "new policy ${request.policyID} has an ism template with index pattern $indexPatterns " + - "matching existing templates ${overlaps.entries.stream().map { "${it.key} => ${it.value}" }.collect( + "matching existing policy templates ${overlaps.entries.stream().map { "policy [${it.key}] => ${it.value}" }.collect( Collectors.joining(","))}," + " please use a different priority than $priority" actionListener.onFailure(IndexManagementException.wrap(IllegalArgumentException(esg))) diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt index 5b6f39cde..d103c3a98 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt @@ -46,7 +46,7 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { } catch (e: ResponseException) { assertEquals("Unexpected RestStatus", RestStatus.BAD_REQUEST, e.response.restStatus()) val actualMessage = e.response.asMap()["error"] as Map<String, Any> - val expectedReason = "new policy $policyID2 has an ism template with index pattern [log*] matching existing templates $policyID1 => [log*], please use a different priority than 100" + val expectedReason = "new policy $policyID2 has an ism template with index pattern [log*] matching existing policy templates policy [$policyID1] => [log*], please use a different priority than 100" assertEquals(expectedReason, actualMessage["reason"]) } } From 77f9d66a40dcdb50d0a5b3b779048fbbce2ac16f Mon Sep 17 00:00:00 2001 From: bowenlan-amzn <bowenlan@amazon.com> Date: Tue, 26 Jan 2021 13:42:29 -0800 Subject: [PATCH 17/21] address Drew's comments --- .../ISMTemplateService.kt | 185 +++++++++--------- .../ManagedIndexCoordinator.kt | 16 +- .../elasticapi/ElasticExtensions.kt | 2 + .../indexstatemanagement/model/ISMTemplate.kt | 3 +- .../indexpolicy/TransportIndexPolicyAction.kt | 17 +- .../util/IndexManagementException.kt | 15 ++ .../mappings/opendistro-ism-config.json | 3 - .../IndexStateManagementRestTestCase.kt | 1 - .../coordinator/ManagedIndexCoordinatorIT.kt | 4 + .../resthandler/ISMTemplateRestAPIIT.kt | 15 ++ .../cached-opendistro-ism-config.json | 3 - 11 files changed, 149 insertions(+), 115 deletions(-) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt index eafd23e7c..7d5ecc049 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt @@ -26,112 +26,107 @@ import org.elasticsearch.common.Strings import org.elasticsearch.common.ValidationException import org.elasticsearch.common.regex.Regex -private val log = LogManager.getLogger(ISMTemplateService::class.java) +private val log = LogManager.getLogger("ISMTemplateService") -class ISMTemplateService { - companion object { - /** - * find the matching policy based on ISM template field for the given index - * - * filter out hidden index - * filter out older index than template lastUpdateTime - * - * @param ismTemplates current ISM templates saved in metadata - * @param indexMetadata cluster state index metadata - * @return policyID - */ - @Suppress("ReturnCount") - fun findMatchingPolicy(ismTemplates: Map<String, ISMTemplate>, indexMetadata: IndexMetadata): String? { - if (ismTemplates.isEmpty()) return null - - val indexName = indexMetadata.index.name +/** + * find the matching policy based on ISM template field for the given index + * + * filter out hidden index + * filter out older index than template lastUpdateTime + * + * @param ismTemplates current ISM templates saved in metadata + * @param indexMetadata cluster state index metadata + * @return policyID + */ +fun Map<String, ISMTemplate>.findMatchingPolicy(indexMetadata: IndexMetadata): String? { + if (this.isEmpty()) return null - // don't include hidden index - val isHidden = IndexMetadata.INDEX_HIDDEN_SETTING.get(indexMetadata.settings) - if (isHidden) return null + val indexName = indexMetadata.index.name - // only process indices created after template - // traverse all ism templates for matching ones - val patternMatchPredicate = { pattern: String -> Regex.simpleMatch(pattern, indexName) } - var matchedPolicy: String? = null - var highestPriority: Int = -1 - ismTemplates.filter { (_, template) -> - template.lastUpdatedTime.toEpochMilli() < indexMetadata.creationDate - }.forEach { (policyID, template) -> - val matched = template.indexPatterns.stream().anyMatch(patternMatchPredicate) - if (matched && highestPriority < template.priority) { - highestPriority = template.priority - matchedPolicy = policyID - } - } + // don't include hidden index + val isHidden = IndexMetadata.INDEX_HIDDEN_SETTING.get(indexMetadata.settings) + if (isHidden) return null - return matchedPolicy + // only process indices created after template + // traverse all ism templates for matching ones + val patternMatchPredicate = { pattern: String -> Regex.simpleMatch(pattern, indexName) } + var matchedPolicy: String? = null + var highestPriority: Int = -1 + this.filter { (_, template) -> + template.lastUpdatedTime.toEpochMilli() < indexMetadata.creationDate + }.forEach { (policyID, template) -> + val matched = template.indexPatterns.stream().anyMatch(patternMatchPredicate) + if (matched && highestPriority < template.priority) { + highestPriority = template.priority + matchedPolicy = policyID } + } + + return matchedPolicy +} - /** - * validate the template Name and indexPattern provided in the template - * reusing ES validate function in MetadataIndexTemplateService - */ - @Suppress("ComplexMethod") - fun validateFormat(indexPatterns: List<String>): ElasticsearchException? { - val validationErrors = mutableListOf<String>() - for (indexPattern in indexPatterns) { - if (indexPattern.contains(" ")) { - validationErrors.add("index_patterns [$indexPattern] must not contain a space") - } - if (indexPattern.contains(",")) { - validationErrors.add("index_pattern [$indexPattern] must not contain a ','") - } - if (indexPattern.contains("#")) { - validationErrors.add("index_pattern [$indexPattern] must not contain a '#'") - } - if (indexPattern.contains(":")) { - validationErrors.add("index_pattern [$indexPattern] must not contain a ':'") - } - if (indexPattern.startsWith("_")) { - validationErrors.add("index_pattern [$indexPattern] must not start with '_'") - } - if (!Strings.validFileNameExcludingAstrix(indexPattern)) { - validationErrors.add("index_pattern [" + indexPattern + "] must not contain the following characters " + - Strings.INVALID_FILENAME_CHARS) - } - } - if (validationErrors.size > 0) { - val validationException = ValidationException() - validationException.addValidationErrors(validationErrors) - return IndexManagementException.wrap(validationException) - } - return null +/** + * validate the template Name and indexPattern provided in the template + * reusing ES validate function in MetadataIndexTemplateService + */ +@Suppress("ComplexMethod") +fun validateFormat(indexPatterns: List<String>): ElasticsearchException? { + val validationErrors = mutableListOf<String>() + for (indexPattern in indexPatterns) { + if (indexPattern.contains(" ")) { + validationErrors.add("index_patterns [$indexPattern] must not contain a space") + } + if (indexPattern.contains(",")) { + validationErrors.add("index_pattern [$indexPattern] must not contain a ','") + } + if (indexPattern.contains("#")) { + validationErrors.add("index_pattern [$indexPattern] must not contain a '#'") } + if (indexPattern.contains(":")) { + validationErrors.add("index_pattern [$indexPattern] must not contain a ':'") + } + if (indexPattern.startsWith("_")) { + validationErrors.add("index_pattern [$indexPattern] must not start with '_'") + } + if (!Strings.validFileNameExcludingAstrix(indexPattern)) { + validationErrors.add("index_pattern [" + indexPattern + "] must not contain the following characters " + + Strings.INVALID_FILENAME_CHARS) + } + } - /** - * find policy templates whose index patterns overlap with given template - * - * @return map of overlapping template name to its index patterns - */ - @Suppress("SpreadOperator") - fun findConflictingPolicyTemplates( - candidate: String, - indexPatterns: List<String>, - priority: Int, - ismTemplates: Map<String, ISMTemplate?> - ): Map<String, List<String>> { - val automaton1 = Regex.simpleMatchToAutomaton(*indexPatterns.toTypedArray()) - val overlappingTemplates = mutableMapOf<String, List<String>>() + if (validationErrors.size > 0) { + val validationException = ValidationException() + validationException.addValidationErrors(validationErrors) + return IndexManagementException.wrap(validationException) + } + return null +} - // focus on template with same priority - ismTemplates.filterNotNullValues() - .filter { it.value.priority == priority }.forEach { (policyID, template) -> - val automaton2 = Regex.simpleMatchToAutomaton(*template.indexPatterns.toTypedArray()) - if (!Operations.isEmpty(Operations.intersection(automaton1, automaton2))) { - log.info("existing ism_template in $policyID overlaps candidate $candidate") - overlappingTemplates[policyID] = template.indexPatterns - } - } - overlappingTemplates.remove(candidate) +/** + * find policy templates whose index patterns overlap with given template + * + * @return map of overlapping template name to its index patterns + */ +@Suppress("SpreadOperator") +fun Map<String, ISMTemplate>.findConflictingPolicyTemplates( + candidate: String, + indexPatterns: List<String>, + priority: Int +): Map<String, List<String>> { + val automaton1 = Regex.simpleMatchToAutomaton(*indexPatterns.toTypedArray()) + val overlappingTemplates = mutableMapOf<String, List<String>>() - return overlappingTemplates + // focus on template with same priority + this.filter { it.value.priority == priority } + .forEach { (policyID, template) -> + val automaton2 = Regex.simpleMatchToAutomaton(*template.indexPatterns.toTypedArray()) + if (!Operations.isEmpty(Operations.intersection(automaton1, automaton2))) { + log.info("existing ism_template in $policyID overlaps candidate $candidate") + overlappingTemplates[policyID] = template.indexPatterns } } + overlappingTemplates.remove(candidate) + + return overlappingTemplates } diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt index eea930039..580ee8236 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt @@ -19,7 +19,6 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlug import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementIndices import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.parseWithType -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ISMTemplateService.Companion.findMatchingPolicy import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.getClusterStateManagedIndexConfig import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.getManagedIndexMetaData import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.getPolicyID @@ -270,12 +269,14 @@ class ManagedIndexCoordinator( * */ var hasCreateRequests = false val updateManagedIndicesRequests = mutableListOf<DocWriteRequest<*>>() + val indicesWithPolicyID = mutableListOf<String>() val indicesToRemoveManagedIndexMetaDataFrom = mutableListOf<Index>() event.state().metadata().indices().forEach { val previousIndexMetaData = event.previousState().metadata().index(it.value.index) val policyID = it.value.getPolicyID() val request: DocWriteRequest<*>? = when { it.value.shouldCreateManagedIndexConfig(previousIndexMetaData) && policyID != null -> { + indicesWithPolicyID.add(it.value.index.name) hasCreateRequests = true managedIndexConfigIndexRequest(it.value.index.name, it.value.indexUUID, policyID, jobInterval) } @@ -289,8 +290,12 @@ class ManagedIndexCoordinator( if (it.value.shouldDeleteManagedIndexMetaData()) indicesToRemoveManagedIndexMetaDataFrom.add(it.value.index) } - // check if newly created indices matching any ISM templates - val updateMatchingIndexReq = getMatchingIndicesUpdateReq(event.state(), event.indicesCreated()) + // Check if newly created indices matching any ISM templates + var updateMatchingIndexReq = emptyList<DocWriteRequest<*>>() + // filter out indices with policyID, they will be picked up in previous block + val indicesCreated = event.indicesCreated().filter { it !in indicesWithPolicyID } + if (indicesCreated.isNotEmpty()) // only check template match if there are new created indices + updateMatchingIndexReq = getMatchingIndicesUpdateReq(event.state(), indicesCreated) if (updateMatchingIndexReq.isNotEmpty()) hasCreateRequests = true updateManagedIndices(updateManagedIndicesRequests + updateMatchingIndexReq + indicesDeletedRequests, hasCreateRequests) @@ -305,7 +310,7 @@ class ManagedIndexCoordinator( val templates = getISMTemplates() val indexToMatchedPolicy = indexNames.map { indexName -> - indexName to findMatchingPolicy(templates, indexMetadatas[indexName]) + indexName to templates.findMatchingPolicy(indexMetadatas[indexName]) }.toMap() val updateManagedIndexReqs = mutableListOf<DocWriteRequest<*>>() @@ -336,6 +341,9 @@ class ManagedIndexCoordinator( emptyMap() } catch (ex: ClusterBlockException) { emptyMap() + } catch (e: Exception) { + logger.error("Failed to get ISM templates", e) + emptyMap() } } diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/elasticapi/ElasticExtensions.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/elasticapi/ElasticExtensions.kt index 060eb0457..629488264 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/elasticapi/ElasticExtensions.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/elasticapi/ElasticExtensions.kt @@ -112,7 +112,9 @@ fun IndexMetadata.getManagedIndexMetaData(): ManagedIndexMetaData? { * parse search response with this function * * @return map of policyID to ISMTemplate in this policy + * @throws [IllegalArgumentException] */ +@Throws(Exception::class) fun getPolicyToTemplateMap(response: SearchResponse, xContentRegistry: NamedXContentRegistry = NamedXContentRegistry.EMPTY): Map<String, ISMTemplate?> { return response.hits.hits.map { diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplate.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplate.kt index 0173459b7..31382f0af 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplate.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplate.kt @@ -40,7 +40,8 @@ data class ISMTemplate( ) : ToXContentObject, Writeable { init { - require(indexPatterns.isNotEmpty()) { "at least give one index pattern" } + require(priority >= 0) { "Requires priority to be >= 0" } + require(indexPatterns.isNotEmpty()) { "Requires at least one index pattern" } } override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/TransportIndexPolicyAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/TransportIndexPolicyAction.kt index a32d4938a..48828bb08 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/TransportIndexPolicyAction.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/TransportIndexPolicyAction.kt @@ -17,11 +17,12 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanageme import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementIndices import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ISMTemplateService.Companion.findConflictingPolicyTemplates -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ISMTemplateService.Companion.validateFormat +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.filterNotNullValues import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.getPolicyToTemplateMap +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.findConflictingPolicyTemplates import com.amazon.opendistroforelasticsearch.indexmanagement.util.IndexManagementException import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.ISM_TEMPLATE_FIELD +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.validateFormat import com.amazon.opendistroforelasticsearch.indexmanagement.util.IndexUtils import org.apache.logging.log4j.LogManager import org.elasticsearch.ElasticsearchStatusException @@ -111,14 +112,14 @@ class TransportIndexPolicyAction @Inject constructor( client.search(searchRequest, object : ActionListener<SearchResponse> { override fun onResponse(response: SearchResponse) { - val policyToTemplateMap = getPolicyToTemplateMap(response, xContentRegistry) - val overlaps = findConflictingPolicyTemplates(request.policyID, indexPatterns, priority, policyToTemplateMap) - if (overlaps.isNotEmpty()) { - val esg = "new policy ${request.policyID} has an ism template with index pattern $indexPatterns " + - "matching existing policy templates ${overlaps.entries.stream().map { "policy [${it.key}] => ${it.value}" }.collect( + val policyToTemplateMap = getPolicyToTemplateMap(response, xContentRegistry).filterNotNullValues() + val conflictingPolicyTemplates = policyToTemplateMap.findConflictingPolicyTemplates(request.policyID, indexPatterns, priority) + if (conflictingPolicyTemplates.isNotEmpty()) { + val errorMessage = "new policy ${request.policyID} has an ism template with index pattern $indexPatterns " + + "matching existing policy templates ${conflictingPolicyTemplates.entries.stream().map { "policy [${it.key}] => ${it.value}" }.collect( Collectors.joining(","))}," + " please use a different priority than $priority" - actionListener.onFailure(IndexManagementException.wrap(IllegalArgumentException(esg))) + actionListener.onFailure(IndexManagementException.wrap(IllegalArgumentException(errorMessage))) return } diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/util/IndexManagementException.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/util/IndexManagementException.kt index 4f8200db7..7b9526a48 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/util/IndexManagementException.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/util/IndexManagementException.kt @@ -1,3 +1,18 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + package com.amazon.opendistroforelasticsearch.indexmanagement.util import org.elasticsearch.ElasticsearchException diff --git a/src/main/resources/mappings/opendistro-ism-config.json b/src/main/resources/mappings/opendistro-ism-config.json index 02365218b..c664fa39f 100644 --- a/src/main/resources/mappings/opendistro-ism-config.json +++ b/src/main/resources/mappings/opendistro-ism-config.json @@ -445,9 +445,6 @@ }, "ism_template": { "properties": { - "template_name": { - "type": "keyword" - }, "index_patterns": { "type": "keyword" }, diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt index 1976c0c1d..1f9832454 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt @@ -719,7 +719,6 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() val templates = response["ism_templates"] as ArrayList<Map<String, Any?>> templatePredicates.forEachIndexed { ind, (_, predicates) -> - // assertTrue("The template: $name was not found in the response: $response", templates.containsKey(name)) val template = templates[ind] predicates.forEach { (fieldName, predicate) -> assertTrue("The key: $fieldName was not found in the response: $template", template.containsKey(fieldName)) diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/coordinator/ManagedIndexCoordinatorIT.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/coordinator/ManagedIndexCoordinatorIT.kt index 92dc0030b..2737b7782 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/coordinator/ManagedIndexCoordinatorIT.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/coordinator/ManagedIndexCoordinatorIT.kt @@ -33,6 +33,7 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagemen import com.amazon.opendistroforelasticsearch.indexmanagement.waitFor import org.elasticsearch.common.settings.Settings import org.elasticsearch.common.xcontent.XContentType +import org.junit.Assert import java.time.Instant import java.time.temporal.ChronoUnit import java.util.Locale @@ -70,6 +71,9 @@ class ManagedIndexCoordinatorIT : IndexStateManagementRestTestCase() { } fun `test creating index with invalid policy_id`() { + wipeAllODFEIndices() + assertIndexDoesNotExist(INDEX_MANAGEMENT_INDEX) + val indexOne = randomAlphaOfLength(10).toLowerCase(Locale.ROOT) val indexTwo = randomAlphaOfLength(10).toLowerCase(Locale.ROOT) val indexThree = randomAlphaOfLength(10).toLowerCase(Locale.ROOT) diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt index d103c3a98..bf2d0853a 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt @@ -1,3 +1,18 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.resthandler import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase diff --git a/src/test/resources/mappings/cached-opendistro-ism-config.json b/src/test/resources/mappings/cached-opendistro-ism-config.json index 02365218b..c664fa39f 100644 --- a/src/test/resources/mappings/cached-opendistro-ism-config.json +++ b/src/test/resources/mappings/cached-opendistro-ism-config.json @@ -445,9 +445,6 @@ }, "ism_template": { "properties": { - "template_name": { - "type": "keyword" - }, "index_patterns": { "type": "keyword" }, From f275a2976f355a24b32cf055f48e7fb518a8a7b0 Mon Sep 17 00:00:00 2001 From: bowenlan-amzn <bowenlan@amazon.com> Date: Tue, 26 Jan 2021 14:06:39 -0800 Subject: [PATCH 18/21] suppress detekt complain --- .../indexstatemanagement/ISMTemplateService.kt | 3 +-- .../indexstatemanagement/IndexStateManagementHistory.kt | 2 +- .../indexstatemanagement/ManagedIndexRunner.kt | 4 ++-- .../indexstatemanagement/model/ManagedIndexMetaData.kt | 1 + .../indexstatemanagement/model/action/ActionConfig.kt | 1 + .../step/forcemerge/AttemptCallForceMergeStep.kt | 2 +- .../indexstatemanagement/step/snapshot/AttemptSnapshotStep.kt | 3 ++- .../action/changepolicy/TransportChangePolicyAction.kt | 1 + .../action/indexpolicy/TransportIndexPolicyAction.kt | 3 ++- .../refreshanalyzer/RefreshSearchAnalyzerRequest.kt | 1 + .../coordinator/ManagedIndexCoordinatorIT.kt | 1 - 11 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt index 7d5ecc049..546884d1c 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt @@ -15,7 +15,6 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement -import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.elasticapi.filterNotNullValues import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ISMTemplate import com.amazon.opendistroforelasticsearch.indexmanagement.util.IndexManagementException import org.apache.logging.log4j.LogManager @@ -38,6 +37,7 @@ private val log = LogManager.getLogger("ISMTemplateService") * @param indexMetadata cluster state index metadata * @return policyID */ +@Suppress("ReturnCount") fun Map<String, ISMTemplate>.findMatchingPolicy(indexMetadata: IndexMetadata): String? { if (this.isEmpty()) return null @@ -65,7 +65,6 @@ fun Map<String, ISMTemplate>.findMatchingPolicy(indexMetadata: IndexMetadata): S return matchedPolicy } - /** * validate the template Name and indexPattern provided in the template * reusing ES validate function in MetadataIndexTemplateService diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementHistory.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementHistory.kt index 1101ce7a8..700cc233d 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementHistory.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/IndexStateManagementHistory.kt @@ -129,7 +129,7 @@ class IndexStateManagementHistory( return response.isRolledOver } - @Suppress("SpreadOperator", "NestedBlockDepth") + @Suppress("SpreadOperator", "NestedBlockDepth", "ComplexMethod") private fun deleteOldHistoryIndex() { val indexToDelete = mutableListOf<String>() diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt index 626839cc2..177e51e44 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt @@ -190,7 +190,7 @@ object ManagedIndexRunner : ScheduledJobRunner, } } - @Suppress("ReturnCount", "ComplexMethod", "LongMethod") + @Suppress("ReturnCount", "ComplexMethod", "LongMethod", "ComplexCondition") private suspend fun runManagedIndexConfig(managedIndexConfig: ManagedIndexConfig) { // doing a check of local cluster health as we do not want to overload master node with potentially a lot of calls if (clusterIsRed()) { @@ -534,7 +534,7 @@ object ManagedIndexRunner : ScheduledJobRunner, * Initializes the change policy process where we will get the policy using the change policy's policyID, update the [ManagedIndexMetaData] * to reflect the new policy, and save the new policy to the [ManagedIndexConfig] while resetting the change policy to null */ - @Suppress("ReturnCount") + @Suppress("ReturnCount", "ComplexMethod") private suspend fun initChangePolicy( managedIndexConfig: ManagedIndexConfig, managedIndexMetaData: ManagedIndexMetaData, diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ManagedIndexMetaData.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ManagedIndexMetaData.kt index 934c3278e..1a0682d67 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ManagedIndexMetaData.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ManagedIndexMetaData.kt @@ -50,6 +50,7 @@ data class ManagedIndexMetaData( val info: Map<String, Any>? ) : Writeable, ToXContentFragment { + @Suppress("ComplexMethod") fun toMap(): Map<String, String> { val resultMap = mutableMapOf<String, String> () resultMap[INDEX] = index diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/ActionConfig.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/ActionConfig.kt index 1fc118367..64ed6d300 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/ActionConfig.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/ActionConfig.kt @@ -87,6 +87,7 @@ abstract class ActionConfig( // TODO clean up for actionIndex @JvmStatic @Throws(IOException::class) + @Suppress("ComplexMethod") fun fromStreamInput(sin: StreamInput): ActionConfig { val type = sin.readEnum(ActionType::class.java) val actionIndex = sin.readInt() diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/step/forcemerge/AttemptCallForceMergeStep.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/step/forcemerge/AttemptCallForceMergeStep.kt index 52622cee7..c8523dd87 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/step/forcemerge/AttemptCallForceMergeStep.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/step/forcemerge/AttemptCallForceMergeStep.kt @@ -50,7 +50,7 @@ class AttemptCallForceMergeStep( override fun isIdempotent() = false - @Suppress("TooGenericExceptionCaught") + @Suppress("TooGenericExceptionCaught", "ComplexMethod") override suspend fun execute(): AttemptCallForceMergeStep { try { diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/step/snapshot/AttemptSnapshotStep.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/step/snapshot/AttemptSnapshotStep.kt index d3d5ede7d..ca0bd5467 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/step/snapshot/AttemptSnapshotStep.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/step/snapshot/AttemptSnapshotStep.kt @@ -147,7 +147,8 @@ class AttemptSnapshotStep( companion object { const val name = "attempt_snapshot" - fun getBlockedMessage(denyList: List<String>, repoName: String, index: String) = "Snapshot repository [$repoName] is blocked in $denyList [index=$index]" + fun getBlockedMessage(denyList: List<String>, repoName: String, index: String) = + "Snapshot repository [$repoName] is blocked in $denyList [index=$index]" fun getFailedMessage(index: String) = "Failed to create snapshot [index=$index]" fun getFailedConcurrentSnapshotMessage(index: String) = "Concurrent snapshot in progress, retrying next execution [index=$index]" fun getSuccessMessage(index: String) = "Successfully started snapshot [index=$index]" diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/changepolicy/TransportChangePolicyAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/changepolicy/TransportChangePolicyAction.kt index 5248f7f88..f83e3a88d 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/changepolicy/TransportChangePolicyAction.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/changepolicy/TransportChangePolicyAction.kt @@ -73,6 +73,7 @@ class TransportChangePolicyAction @Inject constructor( ChangePolicyHandler(client, listener, request).start() } + @Suppress("TooManyFunctions") inner class ChangePolicyHandler( private val client: NodeClient, private val actionListener: ActionListener<ISMStatusResponse>, diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/TransportIndexPolicyAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/TransportIndexPolicyAction.kt index 48828bb08..db79063ca 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/TransportIndexPolicyAction.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/TransportIndexPolicyAction.kt @@ -116,7 +116,8 @@ class TransportIndexPolicyAction @Inject constructor( val conflictingPolicyTemplates = policyToTemplateMap.findConflictingPolicyTemplates(request.policyID, indexPatterns, priority) if (conflictingPolicyTemplates.isNotEmpty()) { val errorMessage = "new policy ${request.policyID} has an ism template with index pattern $indexPatterns " + - "matching existing policy templates ${conflictingPolicyTemplates.entries.stream().map { "policy [${it.key}] => ${it.value}" }.collect( + "matching existing policy templates ${conflictingPolicyTemplates.entries.stream() + .map { "policy [${it.key}] => ${it.value}" }.collect( Collectors.joining(","))}," + " please use a different priority than $priority" actionListener.onFailure(IndexManagementException.wrap(IllegalArgumentException(errorMessage))) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/refreshanalyzer/RefreshSearchAnalyzerRequest.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/refreshanalyzer/RefreshSearchAnalyzerRequest.kt index 19a0d57e9..fbfa05352 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/refreshanalyzer/RefreshSearchAnalyzerRequest.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/refreshanalyzer/RefreshSearchAnalyzerRequest.kt @@ -20,6 +20,7 @@ import org.elasticsearch.common.io.stream.StreamInput import java.io.IOException class RefreshSearchAnalyzerRequest : BroadcastRequest<RefreshSearchAnalyzerRequest> { + @Suppress("SpreadOperator") constructor(vararg indices: String) : super(*indices) @Throws(IOException::class) diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/coordinator/ManagedIndexCoordinatorIT.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/coordinator/ManagedIndexCoordinatorIT.kt index 2737b7782..b1adc67a1 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/coordinator/ManagedIndexCoordinatorIT.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/coordinator/ManagedIndexCoordinatorIT.kt @@ -33,7 +33,6 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagemen import com.amazon.opendistroforelasticsearch.indexmanagement.waitFor import org.elasticsearch.common.settings.Settings import org.elasticsearch.common.xcontent.XContentType -import org.junit.Assert import java.time.Instant import java.time.temporal.ChronoUnit import java.util.Locale From d5578d0ddc4aeb31728aa9e8ff79f7779987a494 Mon Sep 17 00:00:00 2001 From: bowenlan-amzn <bowenlan@amazon.com> Date: Tue, 26 Jan 2021 16:37:29 -0800 Subject: [PATCH 19/21] add a test for ISMTemplate Writeable --- .../model/ISMTemplateTests.kt | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateTests.kt diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateTests.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateTests.kt new file mode 100644 index 000000000..06a95ef87 --- /dev/null +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/ISMTemplateTests.kt @@ -0,0 +1,27 @@ +package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model + +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.randomISMTemplate +import org.elasticsearch.common.io.stream.InputStreamStreamInput +import org.elasticsearch.common.io.stream.OutputStreamStreamOutput +import org.elasticsearch.test.ESTestCase +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream + +class ISMTemplateTests : ESTestCase() { + + fun `test basic`() { + val expectedISMTemplate = randomISMTemplate() + + roundTripISMTemplate(expectedISMTemplate) + } + + private fun roundTripISMTemplate(expectedISMTemplate: ISMTemplate) { + val baos = ByteArrayOutputStream() + val osso = OutputStreamStreamOutput(baos) + expectedISMTemplate.writeTo(osso) + val input = InputStreamStreamInput(ByteArrayInputStream(baos.toByteArray())) + + val actualISMTemplate = ISMTemplate(input) + assertEquals(expectedISMTemplate, actualISMTemplate) + } +} \ No newline at end of file From f39d371c8f3dbb90dfa59c3a99fad0d989c32beb Mon Sep 17 00:00:00 2001 From: bowenlan-amzn <bowenlan@amazon.com> Date: Thu, 28 Jan 2021 12:50:46 -0800 Subject: [PATCH 20/21] coordinator consistent --- .../indexstatemanagement/ManagedIndexCoordinator.kt | 12 +++++++++--- .../resthandler/IndexStateManagementRestApiIT.kt | 1 + 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt index 580ee8236..910c9e6bb 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt @@ -66,6 +66,7 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse import org.elasticsearch.action.bulk.BackoffPolicy import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.search.SearchPhaseExecutionException import org.elasticsearch.action.search.SearchRequest import org.elasticsearch.action.search.SearchResponse import org.elasticsearch.action.support.IndicesOptions @@ -306,6 +307,9 @@ class ManagedIndexCoordinator( * build requests to create jobs for indices matching ISM templates */ suspend fun getMatchingIndicesUpdateReq(clusterState: ClusterState, indexNames: List<String>): List<DocWriteRequest<*>> { + val updateManagedIndexReqs = mutableListOf<DocWriteRequest<*>>() + if (indexNames.isEmpty()) return updateManagedIndexReqs + val indexMetadatas = clusterState.metadata.indices val templates = getISMTemplates() @@ -313,7 +317,6 @@ class ManagedIndexCoordinator( indexName to templates.findMatchingPolicy(indexMetadatas[indexName]) }.toMap() - val updateManagedIndexReqs = mutableListOf<DocWriteRequest<*>>() indexToMatchedPolicy.filterNotNullValues() .forEach { (index, policyID) -> val indexUuid = indexMetadatas[index].indexUUID @@ -341,6 +344,9 @@ class ManagedIndexCoordinator( emptyMap() } catch (ex: ClusterBlockException) { emptyMap() + } catch (e: SearchPhaseExecutionException) { + logger.error("Failed to get ISM templates: $e") + emptyMap() } catch (e: Exception) { logger.error("Failed to get ISM templates", e) emptyMap() @@ -403,8 +409,8 @@ class ManagedIndexCoordinator( // check all un-managed indices, if matches any ism template val unManagedIndices = clusterService.state().metadata.indices.values().filterNotNull() .filter { it.value.indexUUID !in currentManagedIndices.keys }.map { it.value.index.name } - val updateMatchingIndicesReqs = getMatchingIndicesUpdateReq(clusterService.state(), unManagedIndices) - updateManagedIndices(updateMatchingIndicesReqs, updateMatchingIndicesReqs.isNotEmpty()) + val updateMatchingIndicesReq = getMatchingIndicesUpdateReq(clusterService.state(), unManagedIndices) + updateManagedIndices(updateMatchingIndicesReq, updateMatchingIndicesReq.isNotEmpty()) val clusterStateManagedIndices = sweepClusterState(clusterService.state()) diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/IndexStateManagementRestApiIT.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/IndexStateManagementRestApiIT.kt index a1faadf59..e9b8bf8e3 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/IndexStateManagementRestApiIT.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/IndexStateManagementRestApiIT.kt @@ -308,6 +308,7 @@ class IndexStateManagementRestApiIT : IndexStateManagementRestTestCase() { "policy_id" to policy.id, "last_updated_time" to policy.lastUpdatedTime.toEpochMilli(), "default_state" to policy.defaultState, + "ism_template" to null, "description" to policy.description, "error_notification" to policy.errorNotification, "states" to policy.states.map { From 33b3544826183de648287ee3a0035a8748bb7245 Mon Sep 17 00:00:00 2001 From: bowenlan-amzn <bowenlan@amazon.com> Date: Thu, 28 Jan 2021 15:20:22 -0800 Subject: [PATCH 21/21] address Mo's comments --- .../ISMTemplateService.kt | 26 ++++++++----------- .../resthandler/ISMTemplateRestAPIIT.kt | 7 ++--- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt index 546884d1c..f27e3b088 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ISMTemplateService.kt @@ -67,36 +67,32 @@ fun Map<String, ISMTemplate>.findMatchingPolicy(indexMetadata: IndexMetadata): S /** * validate the template Name and indexPattern provided in the template - * reusing ES validate function in MetadataIndexTemplateService + * + * get the idea from ES validate function in MetadataIndexTemplateService + * acknowledge https://github.com/a2lin who should be the first contributor */ @Suppress("ComplexMethod") fun validateFormat(indexPatterns: List<String>): ElasticsearchException? { - val validationErrors = mutableListOf<String>() + val indexPatternFormatErrors = mutableListOf<String>() for (indexPattern in indexPatterns) { - if (indexPattern.contains(" ")) { - validationErrors.add("index_patterns [$indexPattern] must not contain a space") - } - if (indexPattern.contains(",")) { - validationErrors.add("index_pattern [$indexPattern] must not contain a ','") - } if (indexPattern.contains("#")) { - validationErrors.add("index_pattern [$indexPattern] must not contain a '#'") + indexPatternFormatErrors.add("index_pattern [$indexPattern] must not contain a '#'") } if (indexPattern.contains(":")) { - validationErrors.add("index_pattern [$indexPattern] must not contain a ':'") + indexPatternFormatErrors.add("index_pattern [$indexPattern] must not contain a ':'") } if (indexPattern.startsWith("_")) { - validationErrors.add("index_pattern [$indexPattern] must not start with '_'") + indexPatternFormatErrors.add("index_pattern [$indexPattern] must not start with '_'") } if (!Strings.validFileNameExcludingAstrix(indexPattern)) { - validationErrors.add("index_pattern [" + indexPattern + "] must not contain the following characters " + + indexPatternFormatErrors.add("index_pattern [" + indexPattern + "] must not contain the following characters " + Strings.INVALID_FILENAME_CHARS) } } - if (validationErrors.size > 0) { + if (indexPatternFormatErrors.size > 0) { val validationException = ValidationException() - validationException.addValidationErrors(validationErrors) + validationException.addValidationErrors(indexPatternFormatErrors) return IndexManagementException.wrap(validationException) } return null @@ -121,7 +117,7 @@ fun Map<String, ISMTemplate>.findConflictingPolicyTemplates( .forEach { (policyID, template) -> val automaton2 = Regex.simpleMatchToAutomaton(*template.indexPatterns.toTypedArray()) if (!Operations.isEmpty(Operations.intersection(automaton1, automaton2))) { - log.info("existing ism_template in $policyID overlaps candidate $candidate") + log.info("Existing ism_template for $policyID overlaps candidate $candidate") overlappingTemplates[policyID] = template.indexPatterns } } diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt index bf2d0853a..1241bf898 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt @@ -47,7 +47,7 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { } catch (e: ResponseException) { assertEquals("Unexpected RestStatus", RestStatus.BAD_REQUEST, e.response.restStatus()) val actualMessage = e.response.asMap()["error"] as Map<String, Any> - val expectedReason = "Validation Failed: 1: index_patterns [ ] must not contain a space;2: index_pattern [ ] must not contain the following characters [ , \", *, \\, <, |, ,, >, /, ?];" + val expectedReason = "Validation Failed: 1: index_pattern [ ] must not contain the following characters [ , \", *, \\, <, |, ,, >, /, ?];" assertEquals(expectedReason, actualMessage["reason"]) } } @@ -55,13 +55,14 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { fun `test add template with overlapping index pattern`() { try { val ismTemp = ISMTemplate(listOf("log*"), 100, randomInstant()) + val ismTemp2 = ISMTemplate(listOf("lo*"), 100, randomInstant()) createPolicy(randomPolicy(ismTemplate = ismTemp), policyID1) - createPolicy(randomPolicy(ismTemplate = ismTemp), policyID2) + createPolicy(randomPolicy(ismTemplate = ismTemp2), policyID2) fail("Expect a failure") } catch (e: ResponseException) { assertEquals("Unexpected RestStatus", RestStatus.BAD_REQUEST, e.response.restStatus()) val actualMessage = e.response.asMap()["error"] as Map<String, Any> - val expectedReason = "new policy $policyID2 has an ism template with index pattern [log*] matching existing policy templates policy [$policyID1] => [log*], please use a different priority than 100" + val expectedReason = "new policy $policyID2 has an ism template with index pattern [lo*] matching existing policy templates policy [$policyID1] => [log*], please use a different priority than 100" assertEquals(expectedReason, actualMessage["reason"]) } }