[discover] use field caps api to load field list

src/plugins/data_views/common/data_views/data_views.ts

@@ -225,7 +225,7 @@ export interface DataViewsServicePublicMethods {
    */
   getFieldsForIndexPattern: (
     indexPattern: DataView | DataViewSpec,
-    options?: GetFieldsOptions | undefined
+    options?: Omit<GetFieldsOptions, 'pattern'> | undefined
   ) => Promise<FieldSpec[]>;
   /**
    * Get fields for index pattern string
@@ -511,7 +511,7 @@ export class DataViewsService {
    */
   getFieldsForIndexPattern = async (
     indexPattern: DataView | DataViewSpec,
-    options?: GetFieldsOptions
+    options?: Omit<GetFieldsOptions, 'pattern'>
   ) =>
     this.getFieldsForWildcard({
       type: indexPattern.type,

src/plugins/discover/public/application/main/components/layout/discover_layout.tsx

@@ -52,7 +52,7 @@ import { DiscoverHistogramLayout } from './discover_histogram_layout';
  */
 export const SIDEBAR_CLOSED_KEY = 'discover:sidebarClosed';
 
-const SidebarMemoized = React.memo(DiscoverSidebarResponsive);
+const SidebarMemoized = DiscoverSidebarResponsive;
 const TopNavMemoized = React.memo(DiscoverTopNav);
 
 export function DiscoverLayout({

src/plugins/discover/public/application/main/components/sidebar/discover_sidebar.tsx

@@ -226,6 +226,8 @@ export function DiscoverSidebarComponent({
         };
       }
     }, []);
+  // console.log('#### allFields', allFields);
+  // todo
   const { fieldListFiltersProps, fieldListGroupedProps } = useGroupedFields({
     dataViewId: (!isPlainRecord && selectedDataView?.id) || null, // passing `null` for text-based queries
     allFields,

src/plugins/discover/public/application/main/components/sidebar/discover_sidebar_responsive.tsx

@@ -26,7 +26,10 @@ import type { DataView, DataViewField } from '@kbn/data-views-plugin/public';
 import {
   useExistingFieldsFetcher,
   useQuerySubscriber,
+  loadFieldExisting,
 } from '@kbn/unified-field-list-plugin/public';
+import type { AggregateQuery, EsQueryConfig, Filter, Query } from '@kbn/es-query';
+import { getEsQueryConfig } from '@kbn/data-plugin/public';
 import { VIEW_MODE } from '../../../../../common/constants';
 import { useDiscoverServices } from '../../../../hooks/use_discover_services';
 import { DiscoverSidebar } from './discover_sidebar';
@@ -47,6 +50,7 @@ import {
   DiscoverSidebarReducerStatus,
 } from './lib/sidebar_reducer';
 
+const getBuildEsQueryAsync = async () => (await import('@kbn/es-query')).buildEsQuery;
 const EMPTY_FIELD_COUNTS = {};
 
 export interface DiscoverSidebarResponsiveProps {
@@ -136,10 +140,86 @@ export function DiscoverSidebarResponsive(props: DiscoverSidebarResponsiveProps)
   const selectedDataViewRef = useRef<DataView | null | undefined>(selectedDataView);
   const showFieldList = sidebarState.status !== DiscoverSidebarReducerStatus.INITIAL;
 
+  const querySubscriberResult = useQuerySubscriber({ data });
+
+  console.log(
+    '*** DiscoverSidebarResponsive 2 - param:',
+    selectedDataView?.getName(),
+    'ref:',
+    selectedDataViewRef.current?.getName()
+  );
+
+  const fetchFields = useCallback(
+    async (dataView: DataView) => {
+      console.log('*** fetchFields is firing', dataView.getName());
+      const existingFieldList = await loadFieldExisting({
+        data,
+        // dataView: selectedDataViewRef.current!,
+        dataView,
+        fromDate: querySubscriberResult.fromDate || '',
+        toDate: querySubscriberResult.toDate || '',
+        // dslQuery: querySubscriberResult.query || {},
+        dslQuery: await buildSafeEsQuery(
+          // selectedDataViewRef.current!,
+          dataView,
+          querySubscriberResult.query!,
+          querySubscriberResult.filters || [],
+          getEsQueryConfig(core.uiSettings)
+        ),
+        // timeFieldName: selectedDataViewRef.current?.timeFieldName,
+        timeFieldName: dataView.timeFieldName,
+        dataViewsService: dataViews,
+        uiSettingsClient: core.uiSettings,
+      });
+
+      /*
+      console.log(
+        '***** existingFieldList',
+        // selectedDataViewRef.current!.title,
+        dataView.title,
+        JSON.stringify(existingFieldList)
+      );
+      */
+
+      dispatchSidebarStateAction({
+        type: DiscoverSidebarReducerActionType.DOCUMENTS_LOADED,
+        payload: {
+          // dataView: selectedDataViewRef.current,
+          dataView,
+          fieldCounts: existingFieldList.existingFieldNames.reduce((collector, fieldName) => {
+            collector[fieldName] = 1;
+            return collector;
+          }, {} as Record<string, number>),
+          isPlainRecord: false,
+        },
+      });
+    },
+    [
+      core.uiSettings,
+      data,
+      dataViews,
+      querySubscriberResult.fromDate,
+      querySubscriberResult.query,
+      querySubscriberResult.toDate,
+      querySubscriberResult.filters,
+    ]
+  );
+
+  useEffect(() => {
+    selectedDataViewRef.current = selectedDataView;
+    dispatchSidebarStateAction({
+      type: DiscoverSidebarReducerActionType.DATA_VIEW_SWITCHED,
+      payload: {
+        dataView: selectedDataView,
+      },
+    });
+  }, [selectedDataView, dispatchSidebarStateAction]);
+
   useEffect(() => {
     const subscription = props.documents$.subscribe((documentState) => {
       const isPlainRecordType = documentState.recordRawType === RecordRawType.PLAIN;
 
+      // todo - make this parallel to fetch - what triggers new docs?
       switch (documentState?.fetchStatus) {
         case FetchStatus.UNINITIALIZED:
           dispatchSidebarStateAction({
@@ -150,25 +230,34 @@ export function DiscoverSidebarResponsive(props: DiscoverSidebarResponsiveProps)
           });
           break;
         case FetchStatus.LOADING:
-          dispatchSidebarStateAction({
-            type: DiscoverSidebarReducerActionType.DOCUMENTS_LOADING,
-            payload: {
-              isPlainRecord: isPlainRecordType,
-            },
-          });
+          if (isPlainRecordType) {
+            dispatchSidebarStateAction({
+              type: DiscoverSidebarReducerActionType.DOCUMENTS_LOADING,
+              payload: {
+                isPlainRecord: isPlainRecordType,
+              },
+            });
+          } else {
+            console.log('*** 3 - fetchFields', selectedDataViewRef.current);
+            fetchFields(selectedDataViewRef.current!);
+            // fetchFields(selectedDataViewR!);
+          }
           break;
         case FetchStatus.COMPLETE:
-          dispatchSidebarStateAction({
-            type: DiscoverSidebarReducerActionType.DOCUMENTS_LOADED,
-            payload: {
-              dataView: selectedDataViewRef.current,
-              fieldCounts: isPlainRecordType
-                ? EMPTY_FIELD_COUNTS
-                : calcFieldCounts(documentState.result),
-              textBasedQueryColumns: documentState.textBasedQueryColumns,
-              isPlainRecord: isPlainRecordType,
-            },
-          });
+          if (isPlainRecordType) {
+            dispatchSidebarStateAction({
+              type: DiscoverSidebarReducerActionType.DOCUMENTS_LOADED,
+              payload: {
+                dataView: selectedDataViewRef.current,
+                fieldCounts: isPlainRecordType
+                  ? EMPTY_FIELD_COUNTS
+                  : calcFieldCounts(documentState.result),
+                textBasedQueryColumns: documentState.textBasedQueryColumns,
+                isPlainRecord: isPlainRecordType,
+              },
+            });
+          }
+
           break;
         case FetchStatus.ERROR:
           dispatchSidebarStateAction({
@@ -185,7 +274,13 @@ export function DiscoverSidebarResponsive(props: DiscoverSidebarResponsiveProps)
       }
     });
     return () => subscription.unsubscribe();
-  }, [props.documents$, dispatchSidebarStateAction, selectedDataViewRef]);
+  }, [
+    props.documents$,
+    dispatchSidebarStateAction,
+    // selectedDataViewRef,
+    fetchFields,
+    props.useNewFieldsApi,
+  ]);
 
   useEffect(() => {
     if (selectedDataView !== selectedDataViewRef.current) {
@@ -199,11 +294,11 @@ export function DiscoverSidebarResponsive(props: DiscoverSidebarResponsiveProps)
     }
   }, [selectedDataView, dispatchSidebarStateAction, selectedDataViewRef]);
 
-  const querySubscriberResult = useQuerySubscriber({ data });
   const isAffectedByGlobalFilter = Boolean(querySubscriberResult.filters?.length);
+  // this is doing the work
   const { isProcessing, refetchFieldsExistenceInfo } = useExistingFieldsFetcher({
     disableAutoFetching: true,
-    dataViews: !isPlainRecord && sidebarState.dataView ? [sidebarState.dataView] : [],
+    dataViews: !isPlainRecord && selectedDataViewRef.current ? [selectedDataViewRef.current] : [],
     query: querySubscriberResult.query,
     filters: querySubscriberResult.filters,
     fromDate: querySubscriberResult.fromDate,
@@ -217,6 +312,8 @@ export function DiscoverSidebarResponsive(props: DiscoverSidebarResponsiveProps)
 
   useEffect(() => {
     if (sidebarState.status === DiscoverSidebarReducerStatus.COMPLETED) {
+      // todo
+      console.log('*** refetchFieldsExistenceInfo');
       refetchFieldsExistenceInfo();
     }
     // refetching only if status changes
@@ -409,3 +506,26 @@ export function DiscoverSidebarResponsive(props: DiscoverSidebarResponsiveProps)
     </>
   );
 }
+
+// copied from use_existing_fields.ts
+// Wrapper around buildEsQuery, handling errors (e.g. because a query can't be parsed) by
+// returning a query dsl object not matching anything
+async function buildSafeEsQuery(
+  dataView: DataView,
+  query: Query | AggregateQuery,
+  filters: Filter[],
+  queryConfig: EsQueryConfig
+) {
+  const buildEsQuery = await getBuildEsQueryAsync();
+  try {
+    return buildEsQuery(dataView, query, filters, queryConfig);
+  } catch (e) {
+    return {
+      bool: {
+        must_not: {
+          match_all: {},
+        },
+      },
+    };
+  }
+}

src/plugins/discover/public/application/main/components/sidebar/lib/get_field_list.ts

@@ -12,6 +12,7 @@ import type { DatatableColumn } from '@kbn/expressions-plugin/common';
 import { fieldWildcardFilter } from '@kbn/kibana-utils-plugin/public';
 import { isNestedFieldParent } from '../../../utils/nested_fields';
 
+// this applies source filterts and adds unmapped fields
 export function getDataViewFieldList(
   dataView: DataView | undefined | null,
   fieldCounts: Record<string, number> | undefined | null
@@ -21,18 +22,19 @@ export function getDataViewFieldList(
     return null;
   }
 
-  const currentFieldCounts = fieldCounts || {};
+  // console.log('*** getDataViewFieldList', fieldCounts);
+
   const sourceFiltersValues = dataView?.getSourceFiltering?.()?.excludes;
   let dataViewFields: DataViewField[] = dataView?.fields.getAll() || [];
 
   if (sourceFiltersValues) {
     const filter = fieldWildcardFilter(sourceFiltersValues, dataView.metaFields);
     dataViewFields = dataViewFields.filter((field) => {
-      return filter(field.name) || currentFieldCounts[field.name]; // don't filter out a field which was present in hits (ex. for selected fields)
+      return filter(field.name) || fieldCounts[field.name]; // don't filter out a field which was present in hits (ex. for selected fields)
     });
   }
 
-  const fieldNamesInDocs = Object.keys(currentFieldCounts);
+  const fieldNamesInDocs = Object.keys(fieldCounts);
   const fieldNamesInDataView = dataViewFields.map((fld) => fld.name);
   const unknownFields: DataViewField[] = [];
 

src/plugins/discover/public/application/main/components/sidebar/lib/sidebar_reducer.ts

@@ -74,8 +74,10 @@ export function discoverSidebarReducer(
 ): DiscoverSidebarReducerState {
   switch (action.type) {
     case DiscoverSidebarReducerActionType.RESET:
+      // empties field info
       return getInitialState(action.payload.dataView);
     case DiscoverSidebarReducerActionType.DATA_VIEW_SWITCHED:
+      // checks if data view is the same, otherwise empties field info
       return state.dataView === action.payload.dataView
         ? state // already updated in `DOCUMENTS_LOADED`
         : {
@@ -89,16 +91,20 @@ export function discoverSidebarReducer(
                 : state.status,
           };
     case DiscoverSidebarReducerActionType.DOCUMENTS_LOADING:
+      // has a list of fields
       return {
         ...state,
         fieldCounts: null,
+        // what is this?
         allFields: action.payload.isPlainRecord ? null : state.allFields,
         status: DiscoverSidebarReducerStatus.PROCESSING,
       };
     case DiscoverSidebarReducerActionType.DOCUMENTS_LOADED:
+      // todo this
       const mappedAndUnmappedFields = action.payload.isPlainRecord
         ? getTextBasedQueryFieldList(action.payload.textBasedQueryColumns)
         : getDataViewFieldList(action.payload.dataView, action.payload.fieldCounts);
+      // console.log('*** mappedAndUnmappedFields', mappedAndUnmappedFields);
       return {
         ...state,
         dataView: action.payload.dataView,

src/plugins/discover/public/application/main/hooks/use_discover_state.ts

@@ -219,6 +219,7 @@ export function useDiscoverState({
   const onChangeDataView = useCallback(
     async (id: string) => {
       const nextDataView = await dataViews.get(id);
+      console.log('### 1 - onChangeDataView', nextDataView.getName());
       if (nextDataView && dataView) {
         const nextAppState = getDataViewAppState(
           dataView,

src/plugins/discover/public/application/main/utils/calc_field_counts.ts

@@ -20,7 +20,8 @@ export function calcFieldCounts(rows?: DataTableRecord[]) {
   rows.forEach((hit) => {
     const fields = Object.keys(hit.flattened);
     fields.forEach((fieldName) => {
-      counts[fieldName] = (counts[fieldName] || 0) + 1;
+      // todo try setting to 1
+      counts[fieldName] = 1;
     });
   });
 

src/plugins/discover/public/application/main/utils/fetch_all.ts

@@ -62,6 +62,7 @@ export function fetchAll(
   } = fetchDeps;
 
   try {
+    // console.log('*** fetchAll()');
     const dataView = searchSource.getField('index')!;
     if (reset) {
       sendResetMsg(dataSubjects, initialFetchStatus);

src/plugins/unified_field_list/common/utils/field_existing_utils.ts

@@ -53,11 +53,12 @@ export async function fetchFieldExistence({
   const allFields = buildFieldList(dataView, metaFields);
   const existingFieldList = await dataViewsService.getFieldsForIndexPattern(dataView, {
     // filled in by data views service
-    pattern: '',
     indexFilter: toQuery(timeFieldName, fromDate, toDate, dslQuery),
+    includeUnmapped: true,
   });
   return {
-    indexPatternTitle: dataView.title,
+    indexPatternTitle: dataView.getIndexPattern(),
+    // todo - what does this do?
     existingFieldNames: existingFields(existingFieldList, allFields),
   };
 }
@@ -67,6 +68,7 @@ export async function fetchFieldExistence({
  */
 export function buildFieldList(indexPattern: DataView, metaFields: string[]): Field[] {
   return indexPattern.fields.map((field) => {
+    // todo consider removing
     return {
       name: field.name,
       isScript: !!field.scripted,
@@ -124,6 +126,7 @@ export function existingFields(filteredFields: FieldSpec[], allFields: Field[]):
 /**
  * Exported only for unit tests.
  */
+// looks through all the docs to determine which fields are present
 export function legacyExistingFields(docs: estypes.SearchHit[], fields: Field[]): string[] {
   const missingFields = new Set(fields);