Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

createmappings api index pattern support #260

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ public Collection<Object> createComponents(Client client,
Supplier<RepositoriesService> repositoriesServiceSupplier) {
detectorIndices = new DetectorIndices(client.admin(), clusterService, threadPool);
ruleTopicIndices = new RuleTopicIndices(client, clusterService);
mapperService = new MapperService(client.admin().indices(), clusterService);
mapperService = new MapperService(client.admin().indices(), clusterService, indexNameExpressionResolver);
ruleIndices = new RuleIndices(client, clusterService, threadPool);
return List.of(detectorIndices, ruleTopicIndices, ruleIndices, mapperService);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@

import org.opensearch.action.ActionType;

/**
* Acknowledge Alert Action
*/
public class AckAlertsAction extends ActionType<AckAlertsResponse> {
public static final String NAME = "cluster:admin/opensearch/securityanalytics/alerts/ack";
public static final AckAlertsAction INSTANCE = new AckAlertsAction();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@ public AlertsService(Client client) {
*
* @param detectorId id of Detector
* @param table group of search related parameters
* @param severityLevel alert severity level
* @param alertState current alert state
* @param listener ActionListener to get notified on response or error
*/
public void getAlertsByDetectorId(
Expand Down Expand Up @@ -112,8 +114,12 @@ public void onFailure(Exception e) {
/**
* Searches alerts generated by specific Monitor
*
* @param monitorIds id of Monitor
* @param monitorToDetectorMapping monitorId to detectorId mapping
* @param monitorIds list of monitor ids
* @param alertIndex alert index to search alerts on
* @param table group of search related parameters
* @param severityLevel alert severity level
* @param alertState current alert state *
* @param listener ActionListener to get notified on response or error
*/
public void getAlertsByMonitorIds(
Expand Down Expand Up @@ -255,9 +261,9 @@ public void getAlerts(List<String> alertIds,
}

/**
* @param getAlertsResponse
* @param getDetectorResponse
* @param actionListener
* @param getAlertsResponse GetAlerts API response
* @param getDetectorResponse GetDetector API response
* @param actionListener Action Listener
*/
public void ackknowledgeAlerts(org.opensearch.commons.alerting.action.GetAlertsResponse getAlertsResponse,
GetDetectorResponse getDetectorResponse,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ public void onFailure(Exception e) {
* Searches findings generated by specific Monitor
* @param monitorToDetectorMapping monitorId --&gt; detectorId mapper
* @param monitorIds id of Monitor
* @param findingIndexName Finding index name to search findings on
* @param table group of search related parameters
* @param listener ActionListener to get notified on response or error
*/
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,263 @@
/*
Copyright OpenSearch Contributors
SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.securityanalytics.mapper;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.action.ActionListener;
import org.opensearch.action.admin.indices.template.put.PutComponentTemplateAction;
import org.opensearch.action.admin.indices.template.put.PutComposableIndexTemplateAction;
import org.opensearch.action.support.IndicesOptions;
import org.opensearch.action.support.master.AcknowledgedResponse;
import org.opensearch.client.IndicesAdminClient;
import org.opensearch.cluster.ClusterState;
import org.opensearch.cluster.metadata.ComponentTemplate;
import org.opensearch.cluster.metadata.ComposableIndexTemplate;
import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
import org.opensearch.cluster.metadata.MetadataIndexTemplateService;
import org.opensearch.cluster.metadata.Template;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.compress.CompressedXContent;
import org.opensearch.securityanalytics.model.CreateMappingResult;
import org.opensearch.securityanalytics.util.IndexUtils;
import org.opensearch.securityanalytics.util.SecurityAnalyticsException;
import org.opensearch.securityanalytics.util.XContentUtils;

public class IndexTemplateManager {

private static final Logger log = LogManager.getLogger(IndexTemplateManager.class);

private static String OPENSEARCH_SAP_COMPONENT_TEMPLATE_PREFIX = ".opensearch-sap-alias-mappings-component-";
private static String OPENSEARCH_SAP_INDEX_TEMPLATE_PREFIX = ".opensearch-sap-alias-mappings-index-template-";

private IndicesAdminClient indicesClient;
private ClusterService clusterService;
private IndexNameExpressionResolver indexNameExpressionResolver;

public IndexTemplateManager(IndicesAdminClient indicesClient, ClusterService clusterService, IndexNameExpressionResolver indexNameExpressionResolver) {
this.indicesClient = indicesClient;
this.clusterService = clusterService;
this.indexNameExpressionResolver = indexNameExpressionResolver;
}

public void upsertIndexTemplateWithAliasMappings(
String indexName,
Collection<CreateMappingResult> createMappingResults,
ActionListener<AcknowledgedResponse> actionListener
) {
ClusterState state = this.clusterService.state();

if (IndexUtils.isConcreteIndex(indexName, state)) {
actionListener.onFailure(SecurityAnalyticsException.wrap(
new IllegalStateException("Can't upsert index template for concrete index!"))
);
return;
}

String concreteIndexName = IndexUtils.getWriteIndex(indexName, state);
if (concreteIndexName == null) {
String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, IndicesOptions.LENIENT_EXPAND_OPEN, indexName);
if (concreteIndices.length == 0) {
actionListener.onFailure(SecurityAnalyticsException.wrap(
new IllegalStateException("Can't upsert index template for concrete index!"))
);
return;
}
concreteIndexName = IndexUtils.getNewestIndexByCreationDate(concreteIndices, state);
}

// Get applied mappings for our concrete index of interest: writeIndex or newest(creation date)
final String cin = concreteIndexName;
Optional<CreateMappingResult> createMappingResult =
createMappingResults.stream()
.filter(e -> e.getConcreteIndexName().equals(cin))
.findFirst();
if (createMappingResult.isPresent() == false) {
actionListener.onFailure(SecurityAnalyticsException.wrap(
new IllegalStateException("Can't upsert index template for concrete index!"))
);
return;
}

Map<String, Object> mappings = createMappingResult.get().getMappings();

// Upsert component template first
final String index = concreteIndexName;
upsertComponentTemplate(indexName, indicesClient, state, mappings, new ActionListener<>() {
@Override
public void onResponse(AcknowledgedResponse acknowledgedResponse) {

if (acknowledgedResponse.isAcknowledged() == false) {
log.warn("Upserting component template not ack'd!");
}
// Find template which matches input index best
String templateName =
MetadataIndexTemplateService.findV2Template(
state.metadata(),
normalizeIndexName(indexName),
false
);
String componentName = computeComponentTemplateName(indexName);

ComposableIndexTemplate template;
if (templateName == null) {
template = new ComposableIndexTemplate(
List.of(indexName.endsWith("*") == false ? indexName + "*": indexName),
null,
List.of(componentName),
null,
null,
null
);
templateName = computeIndexTemplateName(indexName);
} else {
template = state.metadata().templatesV2().get(templateName);
// Check if we need to append our component to composedOf list
if (template.composedOf().contains(componentName) == false) {
List<String> newComposedOf = new ArrayList<>(template.composedOf());
newComposedOf.add(componentName);
template = new ComposableIndexTemplate(
template.indexPatterns(),
template.template(),
newComposedOf,
template.priority(),
template.version(),
template.metadata(),
template.getDataStreamTemplate()
);
}
}

upsertIndexTemplate(
indicesClient,
templateName == null,
template,
templateName,
actionListener
);
}

@Override
public void onFailure(Exception e) {
actionListener.onFailure(e);
}
});


}

private void upsertIndexTemplate(
IndicesAdminClient indicesClient,
boolean create,
ComposableIndexTemplate indexTemplate,
String templateName,
ActionListener<AcknowledgedResponse> actionListener
) {

indicesClient.execute(
PutComposableIndexTemplateAction.INSTANCE,
new PutComposableIndexTemplateAction.Request(templateName)
.indexTemplate(indexTemplate)
.create(create),
new ActionListener<>() {
@Override
public void onResponse(AcknowledgedResponse acknowledgedResponse) {
actionListener.onResponse(acknowledgedResponse);
}

@Override
public void onFailure(Exception e) {
actionListener.onFailure(e);
}
}
);
}

private void upsertComponentTemplate(
String indexName,
IndicesAdminClient indicesClient,
ClusterState state,
Map<String, Object> mappings,
ActionListener<AcknowledgedResponse> actionListener
) {

String componentName = computeComponentTemplateName(indexName);
boolean create = state.metadata().componentTemplates().containsKey(componentName) == false;
upsertComponentTemplate(componentName, create, indicesClient, mappings, new ActionListener<>() {
@Override
public void onResponse(AcknowledgedResponse acknowledgedResponse) {
actionListener.onResponse(acknowledgedResponse);
}

@Override
public void onFailure(Exception e) {
actionListener.onFailure(e);
}
});
}

private void upsertComponentTemplate(
String componentName,
boolean create,
IndicesAdminClient indicesClient,
Map<String, Object> mappings,
ActionListener<AcknowledgedResponse> actionListener
) {
try {

String mappingsJson = XContentUtils.parseMapToJsonString(mappings);

ComponentTemplate componentTemplate = new ComponentTemplate(
new Template(null, new CompressedXContent(mappingsJson), null),
0L,
null
);
PutComponentTemplateAction.Request req =
new PutComponentTemplateAction.Request(componentName)
.componentTemplate(componentTemplate)
.create(create);

indicesClient.execute(PutComponentTemplateAction.INSTANCE, req, new ActionListener<>() {
@Override
public void onResponse(AcknowledgedResponse acknowledgedResponse) {
actionListener.onResponse(acknowledgedResponse);
}

@Override
public void onFailure(Exception e) {
actionListener.onFailure(e);
}
});
} catch (IOException e) {
actionListener.onFailure(e);
}
}


private static String normalizeIndexName(String indexName) {
if (indexName.endsWith("*")) {
return indexName.substring(0, indexName.length() - 1);
} else {
return indexName;
}
}
public static String computeIndexTemplateName(String indexName) {
return OPENSEARCH_SAP_INDEX_TEMPLATE_PREFIX + normalizeIndexName(indexName);
}

public static String computeComponentTemplateName(String indexName) {
if (indexName.endsWith("*")) {
indexName = indexName.substring(0, indexName.length() - 1);
}
return OPENSEARCH_SAP_COMPONENT_TEMPLATE_PREFIX + normalizeIndexName(indexName);
}
}
Loading