Fix that Trigger page might freeze for high cardinality detectors (#230)

* Fix that Trigger page might freeze for high cardinality detectors

We can define triggers for an AD monitor using AD preview results. The trigger page runs fine for a single-stream detector where the preview results are few. For a high-cardinality detector, the trigger page is likely freezing simply because React needs to draw too many preview results on the page. This PR fixes the issue by holding a worst-case bound on the preview results to show. Specifically, when the number of preview results exceeded the bound, we split the preview time range into small chunks and only kept the maximum anomaly grade results within each chunk. The reduction can keep important results (i.e., the anomalies) intact while speeding up the trigger page rendering.

We have also seen null pointer exceptions during trigger evaluation when the anomaly result index does not exist. The exception can arise when anomaly result indices are deleted by index rollover, and there is no new result index generated. Monitors will send out alerts for the exception. This PR fixes the issue by installing extra null checks.

Testing done:
1. Added/modified unit tests for the above 2 fixes.
2. Manually verified the above 2 issues are fixed.

public/pages/CreateMonitor/components/AnomalyDetectors/AnomaliesChart/AnomaliesChart.js

@@ -28,18 +28,78 @@ import {
 import { ChartContainer } from '../../../../../components/ChartContainer/ChartContainer';
 import DelayedLoader from '../../../../../components/DelayedLoader';
 
+export const MAX_DATA_POINTS = 1000;
+
 const getAxisTitle = (displayGrade, displayConfidence) => {
   if (displayGrade && displayConfidence) {
     return 'Anomaly grade / confidence';
   }
   return displayGrade ? 'Anomaly grade' : 'Anomaly confidence';
 };
 
-const AnomaliesChart = props => {
+/**
+ * In case of too many anomalies (e.g., high-cardinality detectors), we only keep the max anomalies within
+ * allowed range.  Otherwise, return data as they are.
+ * @param {any[]} data The original anomaly result from preview
+ * @returns {any[]} anomalies within allowed range
+ */
+export const prepareDataForChart = (data) => {
+  if (data && data.length > MAX_DATA_POINTS) {
+    return sampleMaxAnomalyGrade(data);
+  } else {
+    return data;
+  }
+};
+
+/**
+ * Caclulate the stride between each step
+ * @param {number} total Total number of preview results
+ * @returns {number} The stride
+ */
+const calculateStep = (total) => {
+  return Math.ceil(total / MAX_DATA_POINTS);
+};
+
+/**
+ * Pick the elememtn with the max anomaly grade within the input array
+ * @param {any[]} anomalies Input array with preview results
+ * @returns The elememtn with the max anomaly grade
+ */
+const findAnomalyWithMaxAnomalyGrade = (anomalies) => {
+  let anomalyWithMaxGrade = anomalies[0];
+  for (let i = 1; i < anomalies.length; i++) {
+    if (anomalies[i].anomalyGrade > anomalyWithMaxGrade.anomalyGrade) {
+      anomalyWithMaxGrade = anomalies[i];
+    }
+  }
+  return anomalyWithMaxGrade;
+};
+
+/**
+ * Sample max anomalies within allowed range
+ * @param {any[]} data The original results from preview
+ * @return {any[]} sampled anomalies
+ */
+const sampleMaxAnomalyGrade = (data) => {
+  let sortedData = data.sort((a, b) => (a.plotTime > b.plotTime ? 1 : -1));
+  const step = calculateStep(sortedData.length);
+  let index = 0;
+  const sampledAnomalies = [];
+  while (index < sortedData.length) {
+    const arr = sortedData.slice(index, index + step);
+    sampledAnomalies.push(findAnomalyWithMaxAnomalyGrade(arr));
+    index = index + step;
+  }
+  return sampledAnomalies;
+};
+
+const AnomaliesChart = (props) => {
   const timeFormatter = niceTimeFormatter([props.startDateTime, props.endDateTime]);
+  const preparedAnomalies = prepareDataForChart(props.anomalies);
+
   return (
     <DelayedLoader isLoading={props.isLoading}>
-      {showLoader => (
+      {(showLoader) => (
         <React.Fragment>
           {props.showTitle ? (
             <EuiText size="xs">
@@ -86,7 +146,7 @@ const AnomaliesChart = props => {
                     yScaleType="linear"
                     xAccessor={'plotTime'}
                     yAccessors={['anomalyGrade']}
-                    data={props.anomalies}
+                    data={preparedAnomalies}
                   />
                 ) : null}
                 {props.displayConfidence ? (
@@ -96,7 +156,7 @@ const AnomaliesChart = props => {
                     yScaleType="linear"
                     xAccessor={'plotTime'}
                     yAccessors={['confidence']}
-                    data={props.anomalies}
+                    data={preparedAnomalies}
                   />
                 ) : null}
               </Chart>

public/pages/CreateMonitor/components/AnomalyDetectors/AnomaliesChart/AnomaliesChart.test.js

@@ -16,11 +16,73 @@
 import React from 'react';
 import { render } from 'enzyme';
 import moment from 'moment';
-import { AnomaliesChart } from './AnomaliesChart';
+import { AnomaliesChart, MAX_DATA_POINTS, prepareDataForChart } from './AnomaliesChart';
 
 const startTime = moment('2018-10-25T09:30:00').valueOf();
 const endTime = moment('2018-10-29T09:30:00').valueOf();
 
+const getRandomArbitrary = (min, max) => {
+  return Math.floor(Math.random() * (max - min) + min);
+};
+
+/**
+ * Generate preview results
+ * @param {Number} startTime Preview start time in epoch milliseconds
+ * @param {Number} endTime Preview end time in epoch milliseconds
+ * @param {Number} count Number of results
+ * @returns {any[]} Generated results
+ */
+const createTestData = (startTime, endTime, count) => {
+  const data = [];
+  const interval = 60000;
+  const midInterval = interval / 2;
+  for (let i = 0; i < count - 3; i++) {
+    let startGenerated = getRandomArbitrary(startTime, endTime);
+    data.push({
+      anomalyGrade: 0,
+      confidence: 0,
+      dataEndTime: startGenerated + interval,
+      dataStartTime: startGenerated,
+      detectorId: 'nxEuT3YBdrEXnzbxJ7XZ',
+      plotTime: startGenerated + midInterval,
+      schemaVersion: 0,
+    });
+  }
+  // injected 3 anomalies: the beginning, the end, and the middle.
+  data.push({
+    anomalyGrade: 1,
+    confidence: 0.7,
+    dataEndTime: startTime + interval,
+    dataStartTime: startTime,
+    detectorId: 'nxEuT3YBdrEXnzbxJ7XZ',
+    plotTime: startTime + midInterval,
+    schemaVersion: 0,
+  });
+
+  data.push({
+    anomalyGrade: 0.9,
+    confidence: 0.8,
+    dataEndTime: endTime,
+    dataStartTime: endTime - interval,
+    detectorId: 'nxEuT3YBdrEXnzbxJ7XZ',
+    plotTime: endTime - interval + midInterval,
+    schemaVersion: 0,
+  });
+
+  let mid = startTime + (endTime - startTime) / 2;
+  data.push({
+    anomalyGrade: 0.7,
+    confidence: 0.9,
+    dataEndTime: mid + interval,
+    dataStartTime: mid,
+    detectorId: 'nxEuT3YBdrEXnzbxJ7XZ',
+    plotTime: mid + midInterval,
+    schemaVersion: 0,
+  });
+
+  return data;
+};
+
 describe('AnomaliesChart', () => {
   test('renders ', () => {
     const sampleData = [
@@ -47,4 +109,40 @@ describe('AnomaliesChart', () => {
     );
     expect(render(component)).toMatchSnapshot();
   });
+
+  test('hc detector trigger definition', () => {
+    let startTime = 1608327992253;
+    let endTime = 1608759992253;
+    const preparedAnomalies = prepareDataForChart(
+      createTestData(startTime, endTime, MAX_DATA_POINTS * 30)
+    );
+
+    expect(preparedAnomalies.length).toBeCloseTo(MAX_DATA_POINTS);
+
+    expect(preparedAnomalies[MAX_DATA_POINTS - 1].anomalyGrade).toBeCloseTo(0.9);
+    expect(preparedAnomalies[MAX_DATA_POINTS - 1].confidence).toBeCloseTo(0.8);
+
+    var anomalyNumber = 0;
+    for (let i = 0; i < MAX_DATA_POINTS; i++) {
+      if (preparedAnomalies[i].anomalyGrade > 0) {
+        anomalyNumber++;
+        // we injected an anomaly in the middle.  Due to randomness, we cannot predict which one it is.
+        if (i > 0 && i < MAX_DATA_POINTS - 1) {
+          expect(preparedAnomalies[i].anomalyGrade).toBeCloseTo(0.7);
+          expect(preparedAnomalies[i].confidence).toBeCloseTo(0.9);
+        }
+      }
+    }
+    // injected 3 anomalies
+    expect(anomalyNumber).toBe(3);
+  });
+
+  test('single-stream detector trigger definition', () => {
+    let startTime = 1608327992253;
+    let endTime = 1608759992253;
+
+    let originalPreviewResults = createTestData(startTime, endTime, MAX_DATA_POINTS);
+    // we only consolidate and reduce original data when the input data size is larger than MAX_DATA_POINTS
+    expect(prepareDataForChart(originalPreviewResults)).toBe(originalPreviewResults);
+  });
 });

public/pages/CreateTrigger/containers/CreateTrigger/utils/constants.js

@@ -43,4 +43,5 @@ export const HITS_TOTAL_RESULTS_PATH = 'ctx.results[0].hits.total.value';
 export const AGGREGATION_RESULTS_PATH = 'ctx.results[0].aggregations.when.value';
 export const ANOMALY_GRADE_RESULT_PATH = 'ctx.results[0].aggregations.max_anomaly_grade.value';
 export const ANOMALY_CONFIDENCE_RESULT_PATH = 'ctx.results[0].hits.hits[0]._source.confidence';
-export const NOT_EMPTY_RESULT = 'ctx.results != null && ctx.results.length > 0';
+export const NOT_EMPTY_RESULT =
+  'ctx.results != null && ctx.results.length > 0 && ctx.results[0].aggregations != null && ctx.results[0].aggregations.max_anomaly_grade != null && ctx.results[0].hits.total.value > 0 && ctx.results[0].hits.hits[0]._source != null && ctx.results[0].hits.hits[0]._source.confidence != null';

public/pages/CreateTrigger/containers/CreateTrigger/utils/formikToTrigger.test.js

@@ -84,7 +84,7 @@ describe('formikToCondition', () => {
       script: {
         lang: 'painless',
         source:
-          'return ctx.results != null && ctx.results.length > 0 && ctx.results[0].aggregations.max_anomaly_grade.value != null && ctx.results[0].aggregations.max_anomaly_grade.value > 0.7 && ctx.results[0].hits.hits[0]._source.confidence > 0.7',
+          'return ctx.results != null && ctx.results.length > 0 && ctx.results[0].aggregations != null && ctx.results[0].aggregations.max_anomaly_grade != null && ctx.results[0].hits.total.value > 0 && ctx.results[0].hits.hits[0]._source != null && ctx.results[0].hits.hits[0]._source.confidence != null && ctx.results[0].aggregations.max_anomaly_grade.value != null && ctx.results[0].aggregations.max_anomaly_grade.value > 0.7 && ctx.results[0].hits.hits[0]._source.confidence > 0.7',
       },
     });
   });

public/pages/CreateTrigger/containers/DefineTrigger/AnomalyDetectorTrigger.js

@@ -33,24 +33,8 @@ class AnomalyDetectorTrigger extends React.Component {
         <AnomalyDetectorData
           detectorId={this.props.detectorId}
           render={(anomalyData) => {
-            let featureData = [];
-            //Skip disabled features showing from Alerting.
-            featureData = get(anomalyData, 'detector.featureAttributes', [])
-              .filter((feature) => feature.featureEnabled)
-              .map((feature, index) => ({
-                featureName: feature.featureName,
-                data: anomalyData.anomalyResult.featureData[feature.featureId] || [],
-              }));
-            const annotations = get(anomalyData, 'anomalyResult.anomalies', [])
-              .filter((anomaly) => anomaly.anomalyGrade > 0)
-              .map((anomaly) => ({
-                coordinates: {
-                  x0: anomaly.startTime,
-                  x1: anomaly.endTime,
-                },
-                details: `There is an anomaly with confidence ${anomaly.confidence}`,
-              }));
-            if (featureData.length > 0) {
+            // using lodash.get without worrying about whether an intermediate property is null or undefined.
+            if (get(anomalyData, 'anomalyResult.anomalies', []).length > 0) {
               return (
                 <React.Fragment>
                   <TriggerExpressions