Lens adhoc dataviews

src/plugins/data/common/query/filters/persistable_state.test.ts

@@ -26,7 +26,7 @@ describe('filter manager persistable state tests', () => {
       const updatedFilters = inject(filters, [
         { type: DATA_VIEW_SAVED_OBJECT_TYPE, name: 'test123', id: '123' },
       ]);
-      expect(updatedFilters[0]).toHaveProperty('meta.index', undefined);
+      expect(updatedFilters[0]).toHaveProperty('meta.index', 'test');
     });
   });
 

src/plugins/data/common/query/filters/persistable_state.ts

@@ -46,7 +46,8 @@ export const inject = (filters: Filter[], references: SavedObjectReference[]) =>
       ...filter,
       meta: {
         ...filter.meta,
-        index: reference && reference.id,
+        // if no reference has been found, keep the current "index" property (used for adhoc data views)
+        index: reference ? reference.id : filter.meta.index,
       },
     };
   });

src/plugins/data/common/query/persistable_state.test.ts

@@ -41,7 +41,7 @@ describe('query service persistable state tests', () => {
       const updatedQueryState = inject(queryState, [
         { type: DATA_VIEW_SAVED_OBJECT_TYPE, name: 'test123', id: '123' },
       ]);
-      expect(updatedQueryState.filters[0]).toHaveProperty('meta.index', undefined);
+      expect(updatedQueryState.filters[0]).toHaveProperty('meta.index', 'test');
     });
   });
 

src/plugins/unified_search/public/dataview_picker/change_dataview.tsx

@@ -69,6 +69,7 @@ export function ChangeDataView({
   onSaveTextLanguageQuery,
   onTextLangQuerySubmit,
   textBasedLanguage,
+  adHocDataViews,
 }: DataViewPickerPropsExtended) {
   const { euiTheme } = useEuiTheme();
   const [isPopoverOpen, setPopoverIsOpen] = useState(false);
@@ -93,10 +94,21 @@ export function ChangeDataView({
   useEffect(() => {
     const fetchDataViews = async () => {
       const dataViewsRefs = await data.dataViews.getIdsWithTitle();
+      if (adHocDataViews?.length) {
+        adHocDataViews.forEach((adHocDataView) => {
+          if (adHocDataView.id) {
+            dataViewsRefs.push({
+              title: adHocDataView.title,
+              name: adHocDataView.name,
+              id: adHocDataView.id,
+            });
+          }
+        });
+      }
       setDataViewsList(dataViewsRefs);
     };
     fetchDataViews();
-  }, [data, currentDataViewId]);
+  }, [data, currentDataViewId, adHocDataViews]);
 
   useEffect(() => {
     if (trigger.label) {

src/plugins/unified_search/public/dataview_picker/index.tsx

@@ -7,6 +7,7 @@
  */
 
 import React from 'react';
+import type { DataView } from '@kbn/data-views-plugin/public';
 import type { EuiButtonProps, EuiSelectableProps } from '@elastic/eui';
 import type { AggregateQuery, Query } from '@kbn/es-query';
 import { ChangeDataView } from './change_dataview';
@@ -44,6 +45,10 @@ export interface DataViewPickerProps {
    * The id of the selected dataview.
    */
   currentDataViewId?: string;
+  /**
+   * The adHocDataview selected.
+   */
+  adHocDataViews?: DataView[];
   /**
    * EuiSelectable properties.
    */
@@ -84,6 +89,7 @@ export interface DataViewPickerPropsExtended extends DataViewPickerProps {
 export const DataViewPicker = ({
   isMissingCurrent,
   currentDataViewId,
+  adHocDataViews,
   onChangeDataView,
   onAddField,
   onDataViewCreated,
@@ -98,6 +104,7 @@ export const DataViewPicker = ({
     <ChangeDataView
       isMissingCurrent={isMissingCurrent}
       currentDataViewId={currentDataViewId}
+      adHocDataViews={adHocDataViews}
       onChangeDataView={onChangeDataView}
       onAddField={onAddField}
       onDataViewCreated={onDataViewCreated}

x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx

@@ -9,6 +9,7 @@ import { isEqual } from 'lodash';
 import { i18n } from '@kbn/i18n';
 import React, { useCallback, useEffect, useMemo, useState, useRef } from 'react';
 import { useStore } from 'react-redux';
+import { asyncForEach } from '@kbn/std';
 import { TopNavMenuData } from '@kbn/navigation-plugin/public';
 import { downloadMultipleAs } from '@kbn/share-plugin/public';
 import { tableHasFormulas } from '@kbn/data-plugin/common';
@@ -251,8 +252,14 @@ export const LensTopNavMenu = ({
     (state: Partial<LensAppState>) => dispatch(setState(state)),
     [dispatch]
   );
+  const [indexPatterns, setIndexPatterns] = useState<DataView[]>([]);
+  const [adHocDataViews, setAdHocDataViews] = useState<DataView[]>();
+  const [currentIndexPattern, setCurrentIndexPattern] = useState<DataView>();
+  const [rejectedIndexPatterns, setRejectedIndexPatterns] = useState<string[]>([]);
+
   const dispatchChangeIndexPattern = React.useCallback(
-    async (indexPatternId) => {
+    async (dataViewOrId) => {
+      const indexPatternId = typeof dataViewOrId === 'string' ? dataViewOrId : dataViewOrId.id;
       const [newIndexPatternRefs, newIndexPatterns] = await Promise.all([
         // Reload refs in case it's a new indexPattern created on the spot
         dataViews.indexPatternRefs[indexPatternId]
@@ -265,9 +272,25 @@ export const LensTopNavMenu = ({
           cache: dataViews.indexPatterns,
         }),
       ]);
+
+      // enhance the references with the adHoc dataviews
+      if (adHocDataViews?.length) {
+        adHocDataViews.forEach((adHoc) => {
+          if (adHoc.id) {
+            newIndexPatternRefs.push({
+              title: adHoc.title,
+              name: adHoc.name,
+              id: adHoc.id,
+            });
+          }
+        });
+      }
       dispatch(
         changeIndexPattern({
-          dataViews: { indexPatterns: newIndexPatterns, indexPatternRefs: newIndexPatternRefs },
+          dataViews: {
+            indexPatterns: newIndexPatterns,
+            indexPatternRefs: newIndexPatternRefs,
+          },
           datasourceIds: Object.keys(datasourceStates),
           visualizationIds: visualization.activeId ? [visualization.activeId] : [],
           indexPatternId,
@@ -281,12 +304,10 @@ export const LensTopNavMenu = ({
       dispatch,
       indexPatternService,
       visualization.activeId,
+      adHocDataViews,
     ]
   );
 
-  const [indexPatterns, setIndexPatterns] = useState<DataView[]>([]);
-  const [currentIndexPattern, setCurrentIndexPattern] = useState<DataView>();
-  const [rejectedIndexPatterns, setRejectedIndexPatterns] = useState<string[]>([]);
   const canEditDataView = Boolean(dataViewEditor?.userPermissions.editDataView());
   const closeFieldEditor = useRef<() => void | undefined>();
   const closeDataViewEditor = useRef<() => void | undefined>();
@@ -344,6 +365,21 @@ export const LensTopNavMenu = ({
     }
   }, [indexPatterns]);
 
+  // add to the dataview picker list the adHoc dataviews
+  useEffect(() => {
+    const setAdHoc = async () => {
+      await asyncForEach(indexPatterns, async (indexPattern) => {
+        if (indexPattern.id && !adHocDataViews?.some((d) => d.id === indexPattern.id)) {
+          const dataViewInstance = await data.dataViews.get(indexPattern.id);
+          if (!dataViewInstance.isPersisted()) {
+            setAdHocDataViews([...(adHocDataViews ?? []), indexPattern]);
+          }
+        }
+      });
+    };
+    setAdHoc();
+  }, [adHocDataViews, data.dataViews, indexPatterns]);
+
   useEffect(() => {
     return () => {
       // Make sure to close the editors when unmounting
@@ -703,14 +739,20 @@ export const LensTopNavMenu = ({
             closeDataViewEditor.current = dataViewEditor.openEditor({
               onSave: async (dataView) => {
                 if (dataView.id) {
-                  dispatchChangeIndexPattern(dataView.id);
+                  dispatchChangeIndexPattern(dataView);
+                  setCurrentIndexPattern(dataView);
+                  if (!dataView.isPersisted()) {
+                    // add the ad-hoc dataview on the indexPatterns list
+                    setIndexPatterns([...indexPatterns, dataView]);
+                  }
                   refreshFieldList();
                 }
               },
+              allowAdHocDataView: true,
             });
           }
         : undefined,
-    [canEditDataView, dataViewEditor, dispatchChangeIndexPattern, refreshFieldList]
+    [canEditDataView, dataViewEditor, dispatchChangeIndexPattern, indexPatterns, refreshFieldList]
   );
 
   const dataViewPickerProps = {
@@ -720,6 +762,7 @@ export const LensTopNavMenu = ({
       title: currentIndexPattern?.title || '',
     },
     currentDataViewId: currentIndexPattern?.id,
+    adHocDataViews,
     onAddField: addField,
     onDataViewCreated: createNewDataView,
     onChangeDataView: (newIndexPatternId: string) => {

x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_helpers.ts

@@ -43,7 +43,8 @@ import { loadIndexPatternRefs, loadIndexPatterns } from '../../indexpattern_serv
 function getIndexPatterns(
   references?: SavedObjectReference[],
   initialContext?: VisualizeFieldContext | VisualizeEditorContext,
-  initialId?: string
+  initialId?: string,
+  adHocDataviews?: string[]
 ) {
   const indexPatternIds = [];
   if (initialContext) {
@@ -67,6 +68,9 @@ function getIndexPatterns(
       }
     }
   }
+  if (adHocDataviews) {
+    indexPatternIds.push(...adHocDataviews);
+  }
   return [...new Set(indexPatternIds)];
 }
 
@@ -113,7 +117,27 @@ export async function initializeDataViews(
       ? fallbackId
       : undefined;
 
-  const usedIndexPatterns = getIndexPatterns(references, initialContext, initialId);
+  const adHocDataviewsIds: string[] = [];
+  let adHocDataviews;
+  Object.keys(datasourceMap).forEach((datasourceId) => {
+    const datasource = datasourceMap[datasourceId];
+    const datasourceState = datasourceStates[datasourceId]?.state;
+    const adHocSpecs = datasource?.getAdHocIndexSpecs?.(datasourceState);
+    if (adHocSpecs) {
+      const dataViewsIds: string[] = Object.keys(adHocSpecs);
+      if (dataViewsIds.length) {
+        adHocDataviewsIds.push(...dataViewsIds);
+        adHocDataviews = Object.values(adHocSpecs);
+      }
+    }
+  });
+
+  const usedIndexPatterns = getIndexPatterns(
+    references,
+    initialContext,
+    initialId,
+    adHocDataviewsIds
+  );
 
   // load them
   const availableIndexPatterns = new Set(indexPatternRefs.map(({ id }: IndexPatternRef) => id));
@@ -125,6 +149,7 @@ export async function initializeDataViews(
     patterns: usedIndexPatterns,
     notUsedPatterns,
     cache: {},
+    adHocDataviews,
   });
 
   return { indexPatternRefs, indexPatterns };

x-pack/plugins/lens/public/embeddable/embeddable.tsx

@@ -76,7 +76,12 @@ import { LensAttributeService } from '../lens_attribute_service';
 import type { ErrorMessage, TableInspectorAdapter } from '../editor_frame_service/types';
 import { getLensInspectorService, LensInspector } from '../lens_inspector_service';
 import { SharingSavedObjectProps, VisualizationDisplayOptions } from '../types';
-import { getActiveDatasourceIdFromDoc, getIndexPatternsObjects, inferTimeField } from '../utils';
+import {
+  getActiveDatasourceIdFromDoc,
+  getIndexPatternsObjects,
+  inferTimeField,
+  getAdHocIndexSpecs,
+} from '../utils';
 import { getLayerMetaInfo, combineQueryAndFilters } from '../app_plugin/show_underlying_data';
 import { convertDataViewIntoLensIndexPattern } from '../indexpattern_service/loader';
 
@@ -770,8 +775,23 @@ export class Embeddable
 
     this.activeDataInfo.activeDatasource = this.deps.datasourceMap[activeDatasourceId];
     const docDatasourceState = this.savedVis?.state.datasourceStates[activeDatasourceId];
+    const adHocIndexPatterns =
+      this.activeDataInfo.activeDatasource?.getAdHocIndexSpecs?.(docDatasourceState);
+
+    const adHocDataviews: DataView[] = [];
+
+    if (adHocIndexPatterns) {
+      const adHocSpecs = Object.values(adHocIndexPatterns);
+      if (adHocSpecs?.length) {
+        for (const addHocDataView of adHocSpecs) {
+          const d = await this.deps.dataViews.create(addHocDataView);
+          adHocDataviews.push(d);
+        }
+      }
+    }
+    const allIndexPatterns = [...this.indexPatterns, ...adHocDataviews];
 
-    const indexPatternsCache = this.indexPatterns.reduce(
+    const indexPatternsCache = allIndexPatterns.reduce(
       (acc, indexPattern) => ({
         [indexPattern.id!]: convertDataViewIntoLensIndexPattern(indexPattern),
         ...acc,
@@ -831,6 +851,19 @@ export class Embeddable
       this.savedVis?.references.map(({ id }) => id) || [],
       this.deps.dataViews
     );
+    const activeDatasourceId = getActiveDatasourceIdFromDoc(this.savedVis);
+    if (activeDatasourceId) {
+      const activeDatasource = this.deps.datasourceMap[activeDatasourceId];
+      const dataSourceState = this.savedVis?.state.datasourceStates[activeDatasourceId];
+      const { adHocDataviews } = await getAdHocIndexSpecs(
+        activeDatasource,
+        dataSourceState,
+        this.deps.dataViews
+      );
+      if (adHocDataviews.length) {
+        indexPatterns.push(...adHocDataviews);
+      }
+    }
 
     this.indexPatterns = uniqBy(indexPatterns, 'id');