[Lens] Introduce 4 new calculation functions: counter rate, cumulative sum, differences, and moving average

x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx

@@ -162,7 +162,7 @@ export function WorkspacePanel({
 
   const expression = useMemo(
     () => {
-      if (!configurationValidationError || configurationValidationError.length === 0) {
+      if (!configurationValidationError?.length) {
         try {
           return buildExpression({
             visualization: activeVisualization,
@@ -400,20 +400,25 @@ export const InnerVisualizationWrapper = ({
         showExtraErrors = localState.configurationValidationError
           .slice(1)
           .map(({ longMessage }) => (
-            <EuiFlexItem key={longMessage} className="eui-textBreakAll">
+            <EuiFlexItem
+              key={longMessage}
+              className="eui-textBreakAll"
+              data-test-subj="configuration-failure-error"
+            >
               {longMessage}
             </EuiFlexItem>
           ));
       } else {
         showExtraErrors = (
-          <EuiFlexItem data-test-subj="configuration-failure-more-errors">
+          <EuiFlexItem>
             <EuiButtonEmpty
               onClick={() => {
                 setLocalState((prevState: WorkspaceState) => ({
                   ...prevState,
                   expandError: !prevState.expandError,
                 }));
               }}
+              data-test-subj="configuration-failure-more-errors"
             >
               {i18n.translate('xpack.lens.editorFrame.configurationFailureMoreErrors', {
                 defaultMessage: ` +{errors} {errors, plural, one {error} other {errors}}`,
@@ -445,7 +450,7 @@ export const InnerVisualizationWrapper = ({
             </EuiTextColor>
           </EuiTitle>
         </EuiFlexItem>
-        <EuiFlexItem className="eui-textBreakAll">
+        <EuiFlexItem className="eui-textBreakAll" data-test-subj="configuration-failure-error">
           {localState.configurationValidationError[0].longMessage}
         </EuiFlexItem>
         {showExtraErrors}

x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx

@@ -16,6 +16,7 @@ import {
   EuiListGroupItemProps,
   EuiFormLabel,
   EuiToolTip,
+  EuiText,
 } from '@elastic/eui';
 import { IndexPatternDimensionEditorProps } from './dimension_panel';
 import { OperationSupportMatrix } from './operation_support';
@@ -37,6 +38,7 @@ import { BucketNestingEditor } from './bucket_nesting_editor';
 import { IndexPattern, IndexPatternLayer } from '../types';
 import { trackUiEvent } from '../../lens_ui_telemetry';
 import { FormatSelector } from './format_selector';
+import { ReferenceEditor } from './reference_editor';
 import { TimeScaling } from './time_scaling';
 
 const operationPanels = getOperationDisplay();
@@ -156,7 +158,10 @@ export function DimensionEditor(props: DimensionEditorProps) {
         (selectedColumn && !hasField(selectedColumn) && definition.input === 'none'),
       disabledStatus:
         definition.getDisabledStatus &&
-        definition.getDisabledStatus(state.indexPatterns[state.currentIndexPatternId]),
+        definition.getDisabledStatus(
+          state.indexPatterns[state.currentIndexPatternId],
+          state.layers[layerId]
+        ),
     };
   });
 
@@ -180,7 +185,15 @@ export function DimensionEditor(props: DimensionEditorProps) {
       }
 
       let label: EuiListGroupItemProps['label'] = operationPanels[operationType].displayName;
-      if (disabledStatus) {
+      if (isActive && disabledStatus) {
+        label = (
+          <EuiToolTip content={disabledStatus} display="block" position="left">
+            <EuiText color="danger" size="s">
+              <strong>{operationPanels[operationType].displayName}</strong>
+            </EuiText>
+          </EuiToolTip>
+        );
+      } else if (disabledStatus) {
         label = (
           <EuiToolTip content={disabledStatus} display="block" position="left">
             <span>{operationPanels[operationType].displayName}</span>
@@ -202,9 +215,12 @@ export function DimensionEditor(props: DimensionEditorProps) {
           compatibleWithCurrentField ? '' : ' incompatible'
         }`,
         onClick() {
-          if (operationDefinitionMap[operationType].input === 'none') {
+          if (
+            operationDefinitionMap[operationType].input === 'none' ||
+            operationDefinitionMap[operationType].input === 'fullReference'
+          ) {
+            // Clear invalid state because we are reseting to a valid column
             if (selectedColumn?.operationType === operationType) {
-              // Clear invalid state because we are reseting to a valid column
               if (incompleteInfo) {
                 setStateWrapper(resetIncomplete(state.layers[layerId], columnId));
               }
@@ -291,6 +307,35 @@ export function DimensionEditor(props: DimensionEditorProps) {
       </div>
       <EuiSpacer size="s" />
       <div className="lnsIndexPatternDimensionEditor__section lnsIndexPatternDimensionEditor__section--shaded">
+        {!incompleteInfo &&
+        selectedColumn &&
+        'references' in selectedColumn &&
+        selectedOperationDefinition?.input === 'fullReference' ? (
+          <>
+            {selectedColumn.references.map((referenceId, index) => {
+              const validation = selectedOperationDefinition.requiredReferences[index];
+
+              return (
+                <ReferenceEditor
+                  key={index}
+                  layer={state.layers[layerId]}
+                  columnId={referenceId}
+                  updateLayer={(newLayer: IndexPatternLayer) => {
+                    setState(mergeLayer({ state, layerId, newLayer }));
+                  }}
+                  validation={validation}
+                  currentIndexPattern={currentIndexPattern}
+                  existingFields={state.existingFields}
+                  selectionStyle={selectedOperationDefinition.selectionStyle}
+                  dateRange={dateRange}
+                  {...services}
+                />
+              );
+            })}
+            <EuiSpacer size="s" />
+          </>
+        ) : null}
+
         {!selectedColumn ||
         selectedOperationDefinition?.input === 'field' ||
         (incompleteOperation && operationDefinitionMap[incompleteOperation].input === 'field') ? (
@@ -325,7 +370,13 @@ export function DimensionEditor(props: DimensionEditorProps) {
               }
               incompleteOperation={incompleteOperation}
               onDeleteColumn={() => {
-                setStateWrapper(deleteColumn({ layer: state.layers[layerId], columnId }));
+                setStateWrapper(
+                  deleteColumn({
+                    layer: state.layers[layerId],
+                    columnId,
+                    indexPattern: currentIndexPattern,
+                  })
+                );
               }}
               onChoose={(choice) => {
                 setStateWrapper(
@@ -342,15 +393,6 @@ export function DimensionEditor(props: DimensionEditorProps) {
           </EuiFormRow>
         ) : null}
 
-        {!currentFieldIsInvalid && !incompleteInfo && selectedColumn && (
-          <TimeScaling
-            selectedColumn={selectedColumn}
-            columnId={columnId}
-            layer={state.layers[layerId]}
-            updateLayer={setStateWrapper}
-          />
-        )}
-
         {!currentFieldIsInvalid && !incompleteInfo && selectedColumn && ParamEditor && (
           <>
             <ParamEditor
@@ -364,6 +406,15 @@ export function DimensionEditor(props: DimensionEditorProps) {
             />
           </>
         )}
+
+        {!currentFieldIsInvalid && !incompleteInfo && selectedColumn && (
+          <TimeScaling
+            selectedColumn={selectedColumn}
+            columnId={columnId}
+            layer={state.layers[layerId]}
+            updateLayer={setStateWrapper}
+          />
+        )}
       </div>
 
       <EuiSpacer size="s" />
@@ -432,11 +483,11 @@ export function DimensionEditor(props: DimensionEditorProps) {
 }
 function getErrorMessage(
   selectedColumn: IndexPatternColumn | undefined,
-  incompatibleSelectedOperationType: boolean,
+  incompleteOperation: boolean,
   input: 'none' | 'field' | 'fullReference' | undefined,
   fieldInvalid: boolean
 ) {
-  if (selectedColumn && incompatibleSelectedOperationType) {
+  if (selectedColumn && incompleteOperation) {
     if (input === 'field') {
       return i18n.translate('xpack.lens.indexPattern.invalidOperationLabel', {
         defaultMessage: 'To use this function, select a different field.',

x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx

@@ -854,6 +854,7 @@ describe('IndexPatternDimensionEditorPanel', () => {
             dataType: 'date',
             isBucketed: true,
             label: '',
+            customLabel: true,
             operationType: 'date_histogram',
             sourceField: 'ts',
             params: {
@@ -872,6 +873,7 @@ describe('IndexPatternDimensionEditorPanel', () => {
         columnId: 'col2',
       };
     }
+
     it('should not show custom options if time scaling is not available', () => {
       wrapper = mount(
         <IndexPatternDimensionEditorComponent
@@ -1149,15 +1151,15 @@ describe('IndexPatternDimensionEditorPanel', () => {
         layers: {
           first: {
             ...state.layers.first,
+            columnOrder: ['col1', 'col2'],
             columns: {
               ...state.layers.first.columns,
               col2: expect.objectContaining({
-                sourceField: 'bytes',
                 operationType: 'avg',
-                // Other parts of this don't matter for this test
+                sourceField: 'bytes',
               }),
             },
-            columnOrder: ['col1', 'col2'],
+            incompleteColumns: {},
           },
         },
       },
@@ -1237,7 +1239,9 @@ describe('IndexPatternDimensionEditorPanel', () => {
   it('should indicate compatible fields when selecting the operation first', () => {
     wrapper = mount(<IndexPatternDimensionEditorComponent {...defaultProps} columnId={'col2'} />);
 
-    wrapper.find('button[data-test-subj="lns-indexPatternDimension-avg"]').simulate('click');
+    act(() => {
+      wrapper.find('button[data-test-subj="lns-indexPatternDimension-avg"]').simulate('click');
+    });
 
     const options = wrapper
       .find(EuiComboBox)
@@ -1317,10 +1321,14 @@ describe('IndexPatternDimensionEditorPanel', () => {
     expect(items.map(({ label }: { label: React.ReactNode }) => label)).toEqual([
       'Average',
       'Count',
+      'Counter rate',
+      'Cumulative sum',
+      'Differences',
       'Last value',
       'Maximum',
       'Median',
       'Minimum',
+      'Moving average',
       'Sum',
       'Unique count',
     ]);
@@ -1536,4 +1544,101 @@ describe('IndexPatternDimensionEditorPanel', () => {
       },
     });
   });
+
+  it('should hide the top level field selector when switching from non-reference to reference', () => {
+    wrapper = mount(<IndexPatternDimensionEditorComponent {...defaultProps} />);
+
+    expect(wrapper.find('ReferenceEditor')).toHaveLength(0);
+
+    wrapper
+      .find('button[data-test-subj="lns-indexPatternDimension-derivative incompatible"]')
+      .simulate('click');
+
+    expect(wrapper.find('ReferenceEditor')).toHaveLength(1);
+  });
+
+  it('should hide the reference editors when switching from reference to non-reference', () => {
+    const stateWithReferences: IndexPatternPrivateState = getStateWithColumns({
+      col1: {
+        label: 'Differences of (incomplete)',
+        dataType: 'number',
+        isBucketed: false,
+        operationType: 'derivative',
+        references: ['col2'],
+        params: {},
+      },
+    });
+
+    wrapper = mount(
+      <IndexPatternDimensionEditorComponent {...defaultProps} state={stateWithReferences} />
+    );
+
+    expect(wrapper.find('ReferenceEditor')).toHaveLength(1);
+
+    wrapper
+      .find('button[data-test-subj="lns-indexPatternDimension-avg incompatible"]')
+      .simulate('click');
+
+    expect(wrapper.find('ReferenceEditor')).toHaveLength(0);
+  });
+
+  it('should show a warning when the current dimension is no longer configurable', () => {
+    const stateWithInvalidCol: IndexPatternPrivateState = getStateWithColumns({
+      col1: {
+        label: 'Invalid derivative',
+        dataType: 'number',
+        isBucketed: false,
+        operationType: 'derivative',
+        references: ['ref1'],
+      },
+    });
+
+    wrapper = mount(
+      <IndexPatternDimensionEditorComponent {...defaultProps} state={stateWithInvalidCol} />
+    );
+
+    expect(
+      wrapper
+        .find('[data-test-subj="lns-indexPatternDimension-derivative incompatible"]')
+        .find('EuiText[color="danger"]')
+        .first()
+    ).toBeTruthy();
+  });
+
+  it('should remove options to select references when there are no time fields', () => {
+    const stateWithoutTime: IndexPatternPrivateState = {
+      ...getStateWithColumns({
+        col1: {
+          label: 'Avg',
+          dataType: 'number',
+          isBucketed: false,
+          operationType: 'avg',
+          sourceField: 'bytes',
+        },
+      }),
+      indexPatterns: {
+        1: {
+          id: '1',
+          title: 'my-fake-index-pattern',
+          hasRestrictions: false,
+          fields,
+          getFieldByName: getFieldByNameFactory([
+            {
+              name: 'bytes',
+              displayName: 'bytes',
+              type: 'number',
+              aggregatable: true,
+              searchable: true,
+            },
+          ]),
+        },
+      },
+    };
+
+    wrapper = mount(
+      <IndexPatternDimensionEditorComponent {...defaultProps} state={stateWithoutTime} />
+    );
+
+    expect(wrapper.find('[data-test-subj="lns-indexPatternDimension-derivative"]')).toHaveLength(0);
+  });
 });

x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/field_select.tsx

@@ -41,6 +41,7 @@ export interface FieldSelectProps extends EuiComboBoxProps<{}> {
   onDeleteColumn: () => void;
   existingFields: IndexPatternPrivateState['existingFields'];
   fieldIsInvalid: boolean;
+  markAllFieldsCompatible?: boolean;
 }
 
 export function FieldSelect({
@@ -53,6 +54,7 @@ export function FieldSelect({
   onDeleteColumn,
   existingFields,
   fieldIsInvalid,
+  markAllFieldsCompatible,
   ...rest
 }: FieldSelectProps) {
   const { operationByField } = operationSupportMatrix;
@@ -93,7 +95,7 @@ export function FieldSelect({
                   : operationByField[field]!.values().next().value,
             },
             exists: containsData(field),
-            compatible: isCompatibleWithCurrentOperation(field),
+            compatible: markAllFieldsCompatible || isCompatibleWithCurrentOperation(field),
           };
         })
         .sort((a, b) => {
@@ -163,6 +165,7 @@ export function FieldSelect({
     currentIndexPattern,
     operationByField,
     existingFields,
+    markAllFieldsCompatible,
   ]);
 
   return (