[ES|QL] METRICS command definition and validation (#184905)

## Summary Partially addresses #184498 The main contribution of this PR is the `METRICS` command validation cases: <img width="778" alt="image" src="https://github.com/elastic/kibana/assets/82822460/3d768952-3fa3-4928-b251-204c30d20c4b"> See own-review below for more comments. ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios ### For maintainers - [x] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
elastic · Jun 20, 2024 · d9fc2ca · d9fc2ca
1 parent 37426f0
commit d9fc2ca
Show file tree

Hide file tree

Showing 31 changed files with 5,201 additions and 4,778 deletions.
diff --git a/packages/kbn-esql-ast/index.ts b/packages/kbn-esql-ast/index.ts
@@ -24,6 +24,7 @@ export type {
   ESQLLiteral,
   AstProviderFn,
   EditorError,
+  ESQLAstNode,
 } from './src/types';
 
 // Low level functions to parse grammar

diff --git a/packages/kbn-esql-ast/src/__tests__/ast_parser.metrics.test.ts b/packages/kbn-esql-ast/src/__tests__/ast_parser.metrics.test.ts
@@ -19,7 +19,7 @@ describe('METRICS', () => {
         {
           type: 'command',
           name: 'metrics',
-          indices: [
+          sources: [
             {
               type: 'source',
               name: 'foo',
@@ -30,7 +30,7 @@ describe('METRICS', () => {
       ]);
     });
 
-    it('can parse multiple "indices"', () => {
+    it('can parse multiple "sources"', () => {
       const text = 'METRICS foo ,\nbar\t,\t\nbaz \n';
       const { ast, errors } = parse(text);
 
@@ -39,7 +39,7 @@ describe('METRICS', () => {
         {
           type: 'command',
           name: 'metrics',
-          indices: [
+          sources: [
             {
               type: 'source',
               name: 'foo',
@@ -69,7 +69,7 @@ describe('METRICS', () => {
         {
           type: 'command',
           name: 'metrics',
-          indices: [
+          sources: [
             {
               type: 'source',
               name: 'foo',
@@ -99,7 +99,7 @@ describe('METRICS', () => {
         {
           type: 'command',
           name: 'metrics',
-          indices: [
+          sources: [
             {
               type: 'source',
               name: 'foo',

diff --git a/packages/kbn-esql-ast/src/ast_factory.ts b/packages/kbn-esql-ast/src/ast_factory.ts
@@ -44,7 +44,7 @@ import {
 import { getPosition } from './ast_position_utils';
 import {
   collectAllSourceIdentifiers,
-  collectAllFieldsStatements,
+  collectAllFields,
   visitByOption,
   collectAllColumnIdentifiers,
   visitRenameClauses,
@@ -120,7 +120,7 @@ export class AstListener implements ESQLParserListener {
   exitRowCommand(ctx: RowCommandContext) {
     const command = createCommand('row', ctx);
     this.ast.push(command);
-    command.args.push(...collectAllFieldsStatements(ctx.fields()));
+    command.args.push(...collectAllFields(ctx.fields()));
   }
 
   /**
@@ -153,20 +153,20 @@ export class AstListener implements ESQLParserListener {
       ...createAstBaseItem('metrics', ctx),
       type: 'command',
       args: [],
-      indices: ctx
+      sources: ctx
         .getTypedRuleContexts(IndexIdentifierContext)
         .map((sourceCtx) => createSource(sourceCtx)),
     };
     this.ast.push(node);
-    const aggregates = collectAllFieldsStatements(ctx.fields(0));
-    const grouping = collectAllFieldsStatements(ctx.fields(1));
+    const aggregates = collectAllFields(ctx.fields(0));
+    const grouping = collectAllFields(ctx.fields(1));
     if (aggregates && aggregates.length) {
       node.aggregates = aggregates;
     }
     if (grouping && grouping.length) {
       node.grouping = grouping;
     }
-    node.args.push(...node.indices, ...aggregates, ...grouping);
+    node.args.push(...node.sources, ...aggregates, ...grouping);
   }
 
   /**
@@ -176,7 +176,7 @@ export class AstListener implements ESQLParserListener {
   exitEvalCommand(ctx: EvalCommandContext) {
     const commandAst = createCommand('eval', ctx);
     this.ast.push(commandAst);
-    commandAst.args.push(...collectAllFieldsStatements(ctx.fields()));
+    commandAst.args.push(...collectAllFields(ctx.fields()));
   }
 
   /**
@@ -189,7 +189,7 @@ export class AstListener implements ESQLParserListener {
 
     // STATS expression is optional
     if (ctx._stats) {
-      command.args.push(...collectAllFieldsStatements(ctx.fields(0)));
+      command.args.push(...collectAllFields(ctx.fields(0)));
     }
     if (ctx._grouping) {
       command.args.push(...visitByOption(ctx, ctx._stats ? ctx.fields(1) : ctx.fields(0)));

diff --git a/packages/kbn-esql-ast/src/ast_walker.ts b/packages/kbn-esql-ast/src/ast_walker.ts
@@ -90,6 +90,7 @@ import type {
   ESQLFunction,
   ESQLCommandOption,
   ESQLAstItem,
+  ESQLAstField,
   ESQLInlineCast,
   ESQLUnnamedParamLiteral,
   ESQLPositionalParamLiteral,
@@ -547,14 +548,14 @@ export function visitField(ctx: FieldContext) {
   return collectBooleanExpression(ctx.booleanExpression());
 }
 
-export function collectAllFieldsStatements(ctx: FieldsContext | undefined): ESQLAstItem[] {
-  const ast: ESQLAstItem[] = [];
+export function collectAllFields(ctx: FieldsContext | undefined): ESQLAstField[] {
+  const ast: ESQLAstField[] = [];
   if (!ctx) {
     return ast;
   }
   try {
     for (const field of ctx.field_list()) {
-      ast.push(...visitField(field));
+      ast.push(...(visitField(field) as ESQLAstField[]));
     }
   } catch (e) {
     // do nothing
@@ -567,7 +568,7 @@ export function visitByOption(ctx: StatsCommandContext, expr: FieldsContext | un
     return [];
   }
   const option = createOption(ctx.BY()!.getText().toLowerCase(), ctx);
-  option.args.push(...collectAllFieldsStatements(expr));
+  option.args.push(...collectAllFields(expr));
   return [option];
 }
 

diff --git a/packages/kbn-esql-ast/src/types.ts b/packages/kbn-esql-ast/src/types.ts
@@ -46,9 +46,9 @@ export interface ESQLCommand<Name = string> extends ESQLAstBaseItem<Name> {
 }
 
 export interface ESQLAstMetricsCommand extends ESQLCommand<'metrics'> {
-  indices: ESQLSource[];
-  aggregates?: ESQLAstItem[];
-  grouping?: ESQLAstItem[];
+  sources: ESQLSource[];
+  aggregates?: ESQLAstField[];
+  grouping?: ESQLAstField[];
 }
 
 export interface ESQLCommandOption extends ESQLAstBaseItem {

diff --git a/packages/kbn-esql-validation-autocomplete/README.md b/packages/kbn-esql-validation-autocomplete/README.md
@@ -39,7 +39,7 @@ const myCallbacks = {
 const { errors, warnings } = await validateQuery("from index | stats 1 + avg(myColumn)", getAstAndSyntaxErrors, undefined, myCallbacks);
 ```
 
-If not all callbacks are available it is possible to gracefully degradate the validation experience with the `ignoreOnMissingCallbacks` option:
+If not all callbacks are available it is possible to gracefully degrade the validation experience with the `ignoreOnMissingCallbacks` option:
 
 ```js
 import { getAstAndSyntaxErrors } from '@kbn/esql-ast';
@@ -61,7 +61,7 @@ const { errors, warnings } = await validateQuery(
 
 #### Autocomplete
 
-This is the complete logic for the ES|QL autocomplete language, it is completely indepedent from the actual editor (i.e. Monaco) and the suggestions reported need to be wrapped against the specific editor shape.
+This is the complete logic for the ES|QL autocomplete language, it is completely independent from the actual editor (i.e. Monaco) and the suggestions reported need to be wrapped against the specific editor shape.
 
 ```js
 import { getAstAndSyntaxErrors } from '@kbn/esql-ast';
@@ -207,13 +207,13 @@ The autocomplete/suggest task takes a query as input together with the current c
 Note that autocomplete works most of the time with incomplete/invalid queries, so some logic to manipulate the query into something valid (see the `EDITOR_MARKER` or the `countBracketsUnclosed` functions for more).
 
 Once the AST is produced there's a `getAstContext` function that finds the cursor position node (and its parent command), together with some hint like the type of current context: `expression`, `function`, `newCommand`, `option`.
-The most complex case is the `expression` as it can cover a moltitude of cases. The function is highly commented in order to identify the specific cases, but there's probably some obscure area still to comment/clarify.
+The most complex case is the `expression` as it can cover a multitude of cases. The function is highly commented in order to identify the specific cases, but there's probably some obscure area still to comment/clarify.
 
-### Adding new commands/options/functions/erc...
+### Adding new commands/options/functions/etc...
 
 To update the definitions:
 
-1. open either approriate definition file within the `definitions` folder and add a new entry to the relative array
+1. open either appropriate definition file within the `definitions` folder and add a new entry to the relative array
 2. if you are adding a function, run `yarn maketests` to add a set of fundamental validation tests for the new definition. If any of the suggested tests are wrong, feel free to correct them by hand. If it seems like a general problem, open an issue with the details so that we can update the generator code.
 3. write new tests for validation and autocomplete
 

diff --git a/packages/kbn-esql-validation-autocomplete/jest.integration.config.js b/packages/kbn-esql-validation-autocomplete/jest.integration.config.js
@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_integration_node',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-esql-validation-autocomplete'],
+  openHandlesTimeout: 0,
+  forceExit: true,
+};
diff --git a/packages/kbn-esql-validation-autocomplete/scripts/generate_function_definitions.ts b/packages/kbn-esql-validation-autocomplete/scripts/generate_function_definitions.ts
@@ -26,7 +26,7 @@ const aliasTable: Record<string, string[]> = {
 const aliases = new Set(Object.values(aliasTable).flat());
 
 const evalSupportedCommandsAndOptions = {
-  supportedCommands: ['stats', 'eval', 'where', 'row', 'sort'],
+  supportedCommands: ['stats', 'metrics', 'eval', 'where', 'row', 'sort'],
   supportedOptions: ['by'],
 };
 
@@ -288,8 +288,24 @@ function printGeneratedFunctionsFile(functionDefinitions: FunctionDefinition[])
 }`;
   };
 
-  const fileHeader = `// NOTE: This file is generated by the generate_function_definitions.ts script
-// Do not edit it manually
+  const fileHeader = `/**
+ * __AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.__
+ *
+ * @note This file is generated by the \`generate_function_definitions.ts\`
+ * script. Do not edit it manually.
+ *
+ *
+ *
+ *
+ *
+ *
+ *
+ *
+ *
+ *
+ *
+ *
+ */
 
 import type { ESQLFunction } from '@kbn/esql-ast';
 import { i18n } from '@kbn/i18n';

diff --git a/packages/kbn-esql-validation-autocomplete/src/definitions/aggs.ts b/packages/kbn-esql-validation-autocomplete/src/definitions/aggs.ts
@@ -28,7 +28,7 @@ function createNumericAggDefinition({
     name,
     type: 'agg',
     description,
-    supportedCommands: ['stats'],
+    supportedCommands: ['stats', 'metrics'],
     signatures: [
       {
         params: [
@@ -98,7 +98,7 @@ export const statsAggregationFunctionDefinitions: FunctionDefinition[] = [
         defaultMessage: 'Returns the maximum value in a field.',
       }),
       type: 'agg',
-      supportedCommands: ['stats'],
+      supportedCommands: ['stats', 'metrics'],
       signatures: [
         {
           params: [{ name: 'column', type: 'number', noNestingFunctions: true }],
@@ -117,7 +117,7 @@ export const statsAggregationFunctionDefinitions: FunctionDefinition[] = [
         defaultMessage: 'Returns the minimum value in a field.',
       }),
       type: 'agg',
-      supportedCommands: ['stats'],
+      supportedCommands: ['stats', 'metrics'],
       signatures: [
         {
           params: [{ name: 'column', type: 'number', noNestingFunctions: true }],
@@ -138,7 +138,7 @@ export const statsAggregationFunctionDefinitions: FunctionDefinition[] = [
       description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.countDoc', {
         defaultMessage: 'Returns the count of the values in a field.',
       }),
-      supportedCommands: ['stats'],
+      supportedCommands: ['stats', 'metrics'],
       signatures: [
         {
           params: [
@@ -164,7 +164,7 @@ export const statsAggregationFunctionDefinitions: FunctionDefinition[] = [
           defaultMessage: 'Returns the count of distinct values in a field.',
         }
       ),
-      supportedCommands: ['stats'],
+      supportedCommands: ['stats', 'metrics'],
       signatures: [
         {
           params: [
@@ -188,7 +188,7 @@ export const statsAggregationFunctionDefinitions: FunctionDefinition[] = [
           defaultMessage: 'Returns the count of distinct values in a field.',
         }
       ),
-      supportedCommands: ['stats'],
+      supportedCommands: ['stats', 'metrics'],
       signatures: [
         {
           params: [{ name: 'column', type: 'cartesian_point', noNestingFunctions: true }],
@@ -212,7 +212,7 @@ export const statsAggregationFunctionDefinitions: FunctionDefinition[] = [
       description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.values', {
         defaultMessage: 'Returns all values in a group as an array.',
       }),
-      supportedCommands: ['stats'],
+      supportedCommands: ['stats', 'metrics'],
       signatures: [
         {
           params: [{ name: 'expression', type: 'any', noNestingFunctions: true }],

diff --git a/packages/kbn-esql-validation-autocomplete/src/definitions/builtin.ts b/packages/kbn-esql-validation-autocomplete/src/definitions/builtin.ts
@@ -22,7 +22,7 @@ function createMathDefinition(
     type: 'builtin',
     name,
     description,
-    supportedCommands: ['eval', 'where', 'row', 'stats', 'sort'],
+    supportedCommands: ['eval', 'where', 'row', 'stats', 'metrics', 'sort'],
     supportedOptions: ['by'],
     signatures: types.map((type) => {
       if (Array.isArray(type)) {
@@ -507,7 +507,7 @@ const otherDefinitions: FunctionDefinition[] = [
     description: i18n.translate('kbn-esql-validation-autocomplete.esql.definition.assignDoc', {
       defaultMessage: 'Assign (=)',
     }),
-    supportedCommands: ['eval', 'stats', 'row', 'dissect', 'where', 'enrich'],
+    supportedCommands: ['eval', 'stats', 'metrics', 'row', 'dissect', 'where', 'enrich'],
     supportedOptions: ['by', 'with'],
     signatures: [
       {

diff --git a/packages/kbn-esql-validation-autocomplete/src/definitions/commands.ts b/packages/kbn-esql-validation-autocomplete/src/definitions/commands.ts
@@ -88,6 +88,36 @@ export const commandDefinitions: CommandDefinition[] = [
       params: [{ name: 'functions', type: 'function' }],
     },
   },
+  {
+    name: 'metrics',
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.metricsDoc', {
+      defaultMessage:
+        'A metrics-specific source command, use this command to load data from TSDB indices. ' +
+        'Similar to STATS command on can calculate aggregate statistics, such as average, count, and sum, over the incoming search results set. ' +
+        'When used without a BY clause, only one row is returned, which is the aggregation over the entire incoming search results set. ' +
+        'When you use a BY clause, one row is returned for each distinct value in the field specified in the BY clause. ' +
+        'The command returns only the fields in the aggregation, and you can use a wide range of statistical functions with the stats command. ' +
+        'When you perform more than one aggregation, separate each aggregation with a comma.',
+    }),
+    examples: [
+      'metrics index',
+      'metrics index, index2',
+      'metrics index avg = avg(a)',
+      'metrics index sum(b) by b',
+      'metrics index, index2 sum(b) by b % 2',
+      'metrics <sources> [ <aggregates> [ by <grouping> ]]',
+      'metrics src1, src2 agg1, agg2 by field1, field2',
+    ],
+    options: [],
+    modes: [],
+    signature: {
+      multipleParams: true,
+      params: [
+        { name: 'index', type: 'source', wildcards: true },
+        { name: 'expression', type: 'function', optional: true },
+      ],
+    },
+  },
   {
     name: 'stats',
     description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.statsDoc', {