[ES|QL] Show common fields source indices and join index (#208681)

## Summary

Partially addresses #206939

This PR introduces the following changes in the `JOIN` command
autocomplete.

Shows intersection of source index and join index fields, moves those
fields to the very top of the list. In the below example the `currency`
field appears in both indices, hence, it is at the very top and with a
different icon:

<img width="786" alt="Screenshot 2025-01-28 at 21 29 52"
src="https://github.com/user-attachments/assets/2c1a058f-80a2-4060-a20e-4a0681043dde"
/>

Adds join index fields to the total list of all fields. In the below
example, the `continenet` field is available only in the joined index,
but it is added to the total list.

<img width="713" alt="Screenshot 2025-01-28 at 21 30 08"
src="https://github.com/user-attachments/assets/7cd44ebf-5fe9-4051-a6eb-3feb28801fa5"
/>


### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

src/platform/packages/shared/kbn-esql-ast/index.ts

@@ -12,6 +12,7 @@ export type {
   ESQLAstItem,
   ESQLAstCommand,
   ESQLAstMetricsCommand,
+  ESQLAstJoinCommand,
   ESQLCommand,
   ESQLCommandOption,
   ESQLCommandMode,

src/platform/packages/shared/kbn-esql-ast/src/mutate/commands/join/index.test.ts

@@ -162,5 +162,22 @@ describe('commands.where', () => {
         },
       ]);
     });
+
+    it('extracts index of an incomplete query', () => {
+      const src = 'FROM kibana_sample_data_ecommerce | LOOKUP JOIN lookup_index ON ';
+      const query = EsqlQuery.fromSrc(src);
+      const summary = commands.join.summarize(query.ast);
+
+      expect(summary).toMatchObject([
+        {
+          target: {
+            index: {
+              type: 'source',
+              name: 'lookup_index',
+            },
+          },
+        },
+      ]);
+    });
   });
 });

src/platform/packages/shared/kbn-esql-ast/src/mutate/commands/join/index.ts

@@ -54,6 +54,41 @@ const getIdentifier = (node: WalkerAstNode): ESQLIdentifier =>
     type: 'identifier',
   }) as ESQLIdentifier;
 
+/**
+ * Summarizes a single JOIN command.
+ *
+ * @param command JOIN command to summarize.
+ * @returns Returns a summary of the JOIN command.
+ */
+export const summarizeCommand = (command: ESQLAstJoinCommand): JoinCommandSummary => {
+  const firstArg = command.args[0];
+  let index: ESQLSource | undefined;
+  let alias: ESQLIdentifier | undefined;
+  const conditions: ESQLAstExpression[] = [];
+
+  if (isAsExpression(firstArg)) {
+    index = getSource(firstArg.args[0]);
+    alias = getIdentifier(firstArg.args[1]);
+  } else {
+    index = getSource(firstArg);
+  }
+
+  const on = generic.commands.options.find(command, ({ name }) => name === 'on');
+
+  conditions.push(...((on?.args || []) as ESQLAstExpression[]));
+
+  const target: JoinCommandTarget = {
+    index: index!,
+    alias,
+  };
+  const summary: JoinCommandSummary = {
+    target,
+    conditions,
+  };
+
+  return summary;
+};
+
 /**
  * Summarizes all JOIN commands in the query.
  *
@@ -65,30 +100,7 @@ export const summarize = (query: ESQLAstQueryExpression): JoinCommandSummary[] =
   const summaries: JoinCommandSummary[] = [];
 
   for (const command of list(query)) {
-    const firstArg = command.args[0];
-    let index: ESQLSource | undefined;
-    let alias: ESQLIdentifier | undefined;
-    const conditions: ESQLAstExpression[] = [];
-
-    if (isAsExpression(firstArg)) {
-      index = getSource(firstArg.args[0]);
-      alias = getIdentifier(firstArg.args[1]);
-    } else {
-      index = getSource(firstArg);
-    }
-
-    const on = generic.commands.options.find(command, ({ name }) => name === 'on');
-
-    conditions.push(...((on?.args || []) as ESQLAstExpression[]));
-
-    const target: JoinCommandTarget = {
-      index: index!,
-      alias,
-    };
-    const summary: JoinCommandSummary = {
-      target,
-      conditions,
-    };
+    const summary = summarizeCommand(command);
 
     summaries.push(summary);
   }

src/platform/packages/shared/kbn-esql-ast/src/pretty_print/leaf_printer.ts

@@ -169,8 +169,11 @@ export const LeafPrinter = {
     return text;
   },
 
-  print: (node: ESQLProperNode): string => {
+  print: (node: ESQLProperNode | ESQLAstComment): string => {
     switch (node.type) {
+      case 'source': {
+        return LeafPrinter.source(node);
+      }
       case 'identifier': {
         return LeafPrinter.identifier(node);
       }
@@ -183,6 +186,9 @@ export const LeafPrinter = {
       case 'timeInterval': {
         return LeafPrinter.timeInterval(node);
       }
+      case 'comment': {
+        return LeafPrinter.comment(node);
+      }
     }
     return '';
   },

src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.command.join.test.ts

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { setup, getFieldNamesByType } from './helpers';
+import { setup, getFieldNamesByType, lookupIndexFields } from './helpers';
 
 describe('autocomplete.suggest', () => {
   describe('<type> JOIN <index> [ AS <alias> ] ON <condition> [, <condition> [, ...]]', () => {
@@ -103,24 +103,34 @@ describe('autocomplete.suggest', () => {
 
       test('suggests fields after ON keyword', async () => {
         const { suggest } = await setup();
-
         const suggestions = await suggest('FROM index | LOOKUP JOIN join_index ON /');
-        const labels = suggestions.map((s) => s.text).sort();
+        const labels = suggestions.map((s) => s.text.trim()).sort();
         const expected = getFieldNamesByType('any')
           .sort()
-          .map((field) => field + ' ');
+          .map((field) => field.trim());
+
+        for (const { name } of lookupIndexFields) {
+          expected.push(name.trim());
+        }
+
+        expected.sort();
 
         expect(labels).toEqual(expected);
       });
 
       test('more field suggestions after comma', async () => {
         const { suggest } = await setup();
-
         const suggestions = await suggest('FROM index | LOOKUP JOIN join_index ON stringField, /');
-        const labels = suggestions.map((s) => s.text).sort();
+        const labels = suggestions.map((s) => s.text.trim()).sort();
         const expected = getFieldNamesByType('any')
           .sort()
-          .map((field) => field + ' ');
+          .map((field) => field.trim());
+
+        for (const { name } of lookupIndexFields) {
+          expected.push(name.trim());
+        }
+
+        expected.sort();
 
         expect(labels).toEqual(expected);
       });

src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/helpers.ts

@@ -48,7 +48,9 @@ export const TIME_PICKER_SUGGESTION: PartialSuggestionWithText = {
 
 export const triggerCharacters = [',', '(', '=', ' '];
 
-export const fields: Array<ESQLRealField & { suggestedAs?: string }> = [
+export type TestField = ESQLRealField & { suggestedAs?: string };
+
+export const fields: TestField[] = [
   ...fieldTypes.map((type) => ({
     name: `${camelCase(type)}Field`,
     type,
@@ -57,6 +59,12 @@ export const fields: Array<ESQLRealField & { suggestedAs?: string }> = [
   { name: 'kubernetes.something.something', type: 'double' },
 ];
 
+export const lookupIndexFields: TestField[] = [
+  { name: 'booleanField', type: 'boolean' },
+  { name: 'dateField', type: 'date' },
+  { name: 'joinIndexOnlyField', type: 'text' },
+];
+
 export const indexes = (
   [] as Array<{ name: string; hidden: boolean; suggestedAs?: string }>
 ).concat(
@@ -279,7 +287,13 @@ export function createCustomCallbackMocks(
   const finalSources = customSources || indexes;
   const finalPolicies = customPolicies || policies;
   return {
-    getColumnsFor: jest.fn(async () => finalColumnsSinceLastCommand),
+    getColumnsFor: jest.fn(async ({ query }) => {
+      if (query === 'FROM join_index') {
+        return lookupIndexFields;
+      }
+
+      return finalColumnsSinceLastCommand;
+    }),
     getSources: jest.fn(async () => finalSources),
     getPolicies: jest.fn(async () => finalPolicies),
     getJoinIndices: jest.fn(async () => ({ indices: joinIndices })),

src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/commands/join/index.ts

@@ -8,17 +8,22 @@
  */
 
 import { i18n } from '@kbn/i18n';
-import { type ESQLAstItem, ESQLAst } from '@kbn/esql-ast';
-import { ESQLCommand } from '@kbn/esql-ast/src/types';
+import { type ESQLAstItem, ESQLAst, ESQLCommand, mutate, LeafPrinter } from '@kbn/esql-ast';
+import type { ESQLAstJoinCommand } from '@kbn/esql-ast';
 import type { ESQLCallbacks } from '../../../shared/types';
 import {
   CommandBaseDefinition,
   CommandDefinition,
   CommandTypeDefinition,
   type SupportedDataType,
 } from '../../../definitions/types';
-import { getPosition, joinIndicesToSuggestions } from './util';
-import { TRIGGER_SUGGESTION_COMMAND } from '../../factories';
+import {
+  getPosition,
+  joinIndicesToSuggestions,
+  suggestionIntersection,
+  suggestionUnion,
+} from './util';
+import { TRIGGER_SUGGESTION_COMMAND, buildFieldsDefinitionsWithMetadata } from '../../factories';
 import type { GetColumnsByTypeFn, SuggestionRawDefinition } from '../../types';
 import { commaCompleteItem, pipeCompleteItem } from '../../complete_items';
 
@@ -37,6 +42,60 @@ const getFullCommandMnemonics = (
   ]);
 };
 
+const suggestFields = async (
+  command: ESQLCommand<'join'>,
+  getColumnsByType: GetColumnsByTypeFn,
+  callbacks?: ESQLCallbacks
+) => {
+  const summary = mutate.commands.join.summarizeCommand(command as ESQLAstJoinCommand);
+  const joinIndexPattern = LeafPrinter.print(summary.target.index);
+
+  const [lookupIndexFields, sourceFields] = await Promise.all([
+    callbacks?.getColumnsFor?.({ query: `FROM ${joinIndexPattern}` }),
+    getColumnsByType(['any'], [], {
+      advanceCursor: true,
+      openSuggestions: true,
+    }),
+  ]);
+
+  const supportsControls = callbacks?.canSuggestVariables?.() ?? false;
+  const getVariablesByType = callbacks?.getVariablesByType;
+  const joinFields = buildFieldsDefinitionsWithMetadata(
+    lookupIndexFields!,
+    { supportsControls },
+    getVariablesByType
+  );
+
+  const intersection = suggestionIntersection(joinFields, sourceFields);
+  const union = suggestionUnion(sourceFields, joinFields);
+
+  for (const commonField of intersection) {
+    commonField.sortText = '1';
+    commonField.documentation = {
+      value: i18n.translate('kbn-esql-validation-autocomplete.esql.autocomplete.join.sharedField', {
+        defaultMessage: 'Field shared between the source and the lookup index',
+      }),
+    };
+
+    let detail = commonField.detail || '';
+
+    if (detail) {
+      detail += ' ';
+    }
+
+    detail += i18n.translate(
+      'kbn-esql-validation-autocomplete.esql.autocomplete.join.commonFieldNote',
+      {
+        defaultMessage: '(common field)',
+      }
+    );
+
+    commonField.detail = detail;
+  }
+
+  return [...intersection, ...union];
+};
+
 export const suggest: CommandBaseDefinition<'join'>['suggest'] = async (
   innerText: string,
   command: ESQLCommand<'join'>,
@@ -113,10 +172,7 @@ export const suggest: CommandBaseDefinition<'join'>['suggest'] = async (
     }
 
     case 'after_on': {
-      const fields = await getColumnsByType(['any'], [], {
-        advanceCursor: true,
-        openSuggestions: true,
-      });
+      const fields = await suggestFields(command, getColumnsByType, callbacks);
 
       return fields;
     }
@@ -127,10 +183,7 @@ export const suggest: CommandBaseDefinition<'join'>['suggest'] = async (
       const commaIsLastToken = !!match?.groups?.comma;
 
       if (commaIsLastToken) {
-        const fields = await getColumnsByType(['any'], [], {
-          advanceCursor: true,
-          openSuggestions: true,
-        });
+        const fields = await suggestFields(command, getColumnsByType, callbacks);
 
         return fields;
       }