Skip to content

Commit

Permalink
Watcher: Ensure all internal search requests count hits (#36697)
Browse files Browse the repository at this point in the history
In previous commits only the stored toXContent version of a search
request was using the old format. However an executed search request was
already disabling hit counts. In 7.0 hit counts will stay enabled by
default to allow for proper migration.

Closes #36177
  • Loading branch information
spinscale authored Dec 18, 2018
1 parent b57e12a commit 00521a5
Show file tree
Hide file tree
Showing 6 changed files with 152 additions and 18 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
---
setup:
- do:
cluster.health:
wait_for_status: yellow

- do:
index:
index: my_test_index
type: doc
id: my_id
refresh: true
body: >
{
"key": "value"
}
---
"Test search input includes hits by default":

- do:
xpack.watcher.execute_watch:
body: >
{
"watch" : {
"trigger": {
"schedule" : { "cron" : "0 0 0 1 * ? 2099" }
},
"input": {
"search" : {
"request" : {
"indices" : [ "my_test_index" ],
"body" : {
"query": {
"match_all" : {}
}
}
}
}
},
"condition": {
"compare": {
"ctx.payload.hits.total": {
"gt": 0
}
}
},
"actions": {
"logging" : {
"logging" : {
"text" : "Logging from a test"
}
}
}
}
}
- match: { watch_record.result.condition.met: true }


---
"Test search transform includes hits by default":

- do:
xpack.watcher.execute_watch:
body: >
{
"watch" : {
"trigger": {
"schedule" : { "cron" : "0 0 0 1 * ? 2099" }
},
"input": {
"simple": {
"foo": "bar"
}
},
"transform" : {
"search" : {
"request" : {
"indices" : [ "my_test_index" ],
"body" : {
"query": {
"match_all" : {}
}
}
}
}
},
"actions": {
"indexme" : {
"condition": {
"compare": {
"ctx.payload.hits.total": {
"gt": 0
}
}
},
"index" : {
"index" : "my_test_index",
"doc_type" : "doc",
"doc_id": "my-id"
}
}
}
}
}
- do:
get:
index: my_test_index
type: doc
id: my_id

- match: { _source.key: "value" }

Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ public class WatcherSearchTemplateRequest implements ToXContentObject {
private final IndicesOptions indicesOptions;
private final Script template;
private final BytesReference searchSource;
private boolean restTotalHitsAsInt;
private boolean restTotalHitsAsInt = true;

public WatcherSearchTemplateRequest(String[] indices, String[] types, SearchType searchType, IndicesOptions indicesOptions,
BytesReference searchSource) {
Expand Down Expand Up @@ -184,7 +184,8 @@ public static WatcherSearchTemplateRequest fromXContent(XContentParser parser, S
IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS;
BytesReference searchSource = null;
Script template = null;
boolean totalHitsAsInt = false;
// TODO this is to retain BWC compatibility in 7.0 and can be removed for 8.0
boolean totalHitsAsInt = true;

XContentParser.Token token;
String currentFieldName = null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

import static java.util.Collections.singletonMap;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;

public class WatcherSearchTemplateRequestTests extends ESTestCase {

Expand All @@ -28,16 +29,33 @@ public void testFromXContentWithTemplateCustomLang() throws IOException {
assertTemplate(source, "custom-script", "painful", singletonMap("bar", "baz"));
}

private void assertTemplate(String source, String expectedScript, String expectedLang, Map<String, Object> expectedParams) {
public void testDefaultHitCountsDefaults() throws IOException {
assertHitCount("{}", true);
}

public void testDefaultHitCountsConfigured() throws IOException {
boolean hitCountsAsInt = randomBoolean();
String source = "{ \"rest_total_hits_as_int\" : " + hitCountsAsInt + " }";
assertHitCount(source, hitCountsAsInt);
}

private void assertHitCount(String source, boolean expectedHitCountAsInt) throws IOException {
try (XContentParser parser = createParser(JsonXContent.jsonXContent, source)) {
parser.nextToken();
WatcherSearchTemplateRequest request = WatcherSearchTemplateRequest.fromXContent(parser, SearchType.QUERY_THEN_FETCH);
assertThat(request.isRestTotalHitsAsint(), is(expectedHitCountAsInt));
}
}

private void assertTemplate(String source, String expectedScript, String expectedLang, Map<String, Object> expectedParams)
throws IOException {
try (XContentParser parser = createParser(JsonXContent.jsonXContent, source)) {
parser.nextToken();
WatcherSearchTemplateRequest result = WatcherSearchTemplateRequest.fromXContent(parser, randomFrom(SearchType.values()));
assertNotNull(result.getTemplate());
assertThat(result.getTemplate().getIdOrCode(), equalTo(expectedScript));
assertThat(result.getTemplate().getLang(), equalTo(expectedLang));
assertThat(result.getTemplate().getParams(), equalTo(expectedParams));
} catch (IOException e) {
fail("Failed to parse watch search request: " + e.getMessage());
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -340,7 +340,7 @@ protected void assertWatchWithMinimumPerformedActionsCount(final String watchNam
assertThat("could not find executed watch record for watch " + watchName, searchResponse.getHits().getTotalHits().value,
greaterThanOrEqualTo(minimumExpectedWatchActionsWithActionPerformed));
if (assertConditionMet) {
assertThat((Integer) XContentMapValues.extractValue("result.input.payload.hits.total.value",
assertThat((Integer) XContentMapValues.extractValue("result.input.payload.hits.total",
searchResponse.getHits().getAt(0).getSourceAsMap()), greaterThanOrEqualTo(1));
}
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ public void testIndexWatch() throws Exception {
.setSource(watchBuilder()
.trigger(schedule(interval(5, IntervalSchedule.Interval.Unit.SECONDS)))
.input(searchInput(request))
.condition(new CompareCondition("ctx.payload.hits.total.value", CompareCondition.Op.EQ, 1L))
.condition(new CompareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 1L))
.addAction("_logger", loggingAction("_logging")
.setCategory("_category")))
.get();
Expand All @@ -95,7 +95,7 @@ public void testIndexWatchRegisterWatchBeforeTargetIndex() throws Exception {
.setSource(watchBuilder()
.trigger(schedule(interval(5, IntervalSchedule.Interval.Unit.SECONDS)))
.input(searchInput(searchRequest))
.condition(new CompareCondition("ctx.payload.hits.total.value", CompareCondition.Op.EQ, 1L)))
.condition(new CompareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 1L)))
.get();
timeWarp().trigger("_name");
// The watch's condition won't meet because there is no data that matches with the query
Expand All @@ -119,7 +119,7 @@ public void testDeleteWatch() throws Exception {
.setSource(watchBuilder()
.trigger(schedule(cron("0/1 * * * * ? 2020")))
.input(searchInput(searchRequest))
.condition(new CompareCondition("ctx.payload.hits.total.value", CompareCondition.Op.EQ, 1L)))
.condition(new CompareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 1L)))
.get();
assertThat(indexResponse.isCreated(), is(true));
DeleteWatchResponse deleteWatchResponse = watcherClient.prepareDeleteWatch("_name").get();
Expand Down Expand Up @@ -180,15 +180,15 @@ public void testModifyWatches() throws Exception {
.addAction("_id", indexAction("idx", "action"));

watcherClient().preparePutWatch("_name")
.setSource(source.condition(new CompareCondition("ctx.payload.hits.total.value", CompareCondition.Op.EQ, 1L)))
.setSource(source.condition(new CompareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 1L)))
.get();

timeWarp().clock().fastForwardSeconds(5);
timeWarp().trigger("_name");
assertWatchWithMinimumPerformedActionsCount("_name", 0, false);

watcherClient().preparePutWatch("_name")
.setSource(source.condition(new CompareCondition("ctx.payload.hits.total.value", CompareCondition.Op.EQ, 0L)))
.setSource(source.condition(new CompareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 0L)))
.get();

timeWarp().clock().fastForwardSeconds(5);
Expand All @@ -199,7 +199,7 @@ public void testModifyWatches() throws Exception {
watcherClient().preparePutWatch("_name")
.setSource(source
.trigger(schedule(Schedules.cron("0/1 * * * * ? 2020")))
.condition(new CompareCondition("ctx.payload.hits.total.value", CompareCondition.Op.EQ, 0L)))
.condition(new CompareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 0L)))
.get();

timeWarp().clock().fastForwardSeconds(5);
Expand Down Expand Up @@ -245,7 +245,7 @@ public void testInputFiltering() throws Exception {
.setSource(watchBuilder()
.trigger(schedule(interval(5, IntervalSchedule.Interval.Unit.SECONDS)))
.input(searchInput(request).extractKeys("hits.total.value"))
.condition(new CompareCondition("ctx.payload.hits.total.value", CompareCondition.Op.EQ, 1L)))
.condition(new CompareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 1L)))
.get();
// in this watcher the condition will fail, because max_score isn't extracted, only total:
watcherClient.preparePutWatch("_name2")
Expand All @@ -265,7 +265,7 @@ public void testInputFiltering() throws Exception {
SearchResponse searchResponse = searchWatchRecords(builder -> builder.setQuery(matchQuery("watch_id", "_name1")));
assertHitCount(searchResponse, 1);
XContentSource source = xContentSource(searchResponse.getHits().getAt(0).getSourceRef());
assertThat(source.getValue("result.input.payload.hits.total.value"), equalTo((Object) 1));
assertThat(source.getValue("result.input.payload.hits.total"), equalTo((Object) 1));
}

public void testPutWatchWithNegativeSchedule() throws Exception {
Expand Down Expand Up @@ -349,7 +349,7 @@ private void testConditionSearch(WatcherSearchTemplateRequest request) throws Ex
.setSource(watchBuilder()
.trigger(schedule(interval("5s")))
.input(searchInput(request))
.condition(new CompareCondition("ctx.payload.hits.total.value", CompareCondition.Op.GTE, 3L)))
.condition(new CompareCondition("ctx.payload.hits.total", CompareCondition.Op.GTE, 3L)))
.get();

logger.info("created watch [{}] at [{}]", watchName, new DateTime(Clock.systemUTC().millis()));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ public void testAckSingleAction() throws Exception {
.setSource(watchBuilder()
.trigger(schedule(cron("0/5 * * * * ? *")))
.input(searchInput(templateRequest(searchSource(), "events")))
.condition(new CompareCondition("ctx.payload.hits.total.value", CompareCondition.Op.GT, 0L))
.condition(new CompareCondition("ctx.payload.hits.total", CompareCondition.Op.GT, 0L))
.transform(searchTransform(templateRequest(searchSource(), "events")))
.addAction("_a1", indexAction("actions1", "doc"))
.addAction("_a2", indexAction("actions2", "doc"))
Expand Down Expand Up @@ -127,7 +127,7 @@ public void testAckAllActions() throws Exception {
.setSource(watchBuilder()
.trigger(schedule(cron("0/5 * * * * ? *")))
.input(searchInput(templateRequest(searchSource(), "events")))
.condition(new CompareCondition("ctx.payload.hits.total.value", CompareCondition.Op.GT, 0L))
.condition(new CompareCondition("ctx.payload.hits.total", CompareCondition.Op.GT, 0L))
.transform(searchTransform(templateRequest(searchSource(), "events")))
.addAction("_a1", indexAction("actions1", "doc"))
.addAction("_a2", indexAction("actions2", "doc"))
Expand Down Expand Up @@ -195,7 +195,7 @@ public void testAckWithRestart() throws Exception {
.setSource(watchBuilder()
.trigger(schedule(cron("0/5 * * * * ? *")))
.input(searchInput(templateRequest(searchSource(), "events")))
.condition(new CompareCondition("ctx.payload.hits.total.value", CompareCondition.Op.GT, 0L))
.condition(new CompareCondition("ctx.payload.hits.total", CompareCondition.Op.GT, 0L))
.transform(searchTransform(templateRequest(searchSource(), "events")))
.addAction("_id", indexAction("actions", "action")))
.get();
Expand Down

0 comments on commit 00521a5

Please sign in to comment.