[Discover] Use fields API to retrieve fields

src/plugins/discover/common/index.ts

@@ -29,3 +29,4 @@ export const CONTEXT_STEP_SETTING = 'context:step';
 export const CONTEXT_TIE_BREAKER_FIELDS_SETTING = 'context:tieBreakerFields';
 export const DOC_TABLE_LEGACY = 'doc_table:legacy';
 export const MODIFY_COLUMNS_ON_SWITCH = 'discover:modifyColumnsOnSwitch';
+export const SEARCH_FIELDS_FROM_SOURCE = 'discover:searchFieldsFromSource';

src/fixtures/stubbed_saved_object_index_pattern.ts → src/plugins/discover/public/__mocks__/stubbed_saved_object_index_pattern.ts

@@ -18,7 +18,7 @@
  */
 
 // @ts-expect-error
-import stubbedLogstashFields from './logstash_fields';
+import stubbedLogstashFields from '../../../../fixtures/logstash_fields';
 
 const mockLogstashFields = stubbedLogstashFields();
 

src/plugins/discover/public/application/angular/context/api/_stubs.js

@@ -47,6 +47,7 @@ export function createSearchSourceStub(hits, timeField) {
 
   searchSourceStub.setParent = sinon.spy(() => searchSourceStub);
   searchSourceStub.setField = sinon.spy(() => searchSourceStub);
+  searchSourceStub.removeField = sinon.spy(() => searchSourceStub);
 
   searchSourceStub.getField = sinon.spy((key) => {
     const previousSetCall = searchSourceStub.setField.withArgs(key).lastCall;

src/plugins/discover/public/application/angular/context/api/anchor.js

@@ -20,7 +20,7 @@
 import _ from 'lodash';
 import { i18n } from '@kbn/i18n';
 
-export function fetchAnchorProvider(indexPatterns, searchSource) {
+export function fetchAnchorProvider(indexPatterns, searchSource, useNewFieldsApi = false) {
   return async function fetchAnchor(indexPatternId, anchorId, sort) {
     const indexPattern = await indexPatterns.get(indexPatternId);
     searchSource
@@ -41,7 +41,10 @@ export function fetchAnchorProvider(indexPatterns, searchSource) {
         language: 'lucene',
       })
       .setField('sort', sort);
-
+    if (useNewFieldsApi) {
+      searchSource.removeField('fieldsFromSource');
+      searchSource.setField('fields', ['*']);
+    }
     const response = await searchSource.fetch();
 
     if (_.get(response, ['hits', 'total'], 0) < 1) {

src/plugins/discover/public/application/angular/context/api/anchor.test.js

@@ -144,4 +144,29 @@ describe('context app', function () {
       });
     });
   });
+
+  describe('useNewFields API', () => {
+    let fetchAnchor;
+    let searchSourceStub;
+
+    beforeEach(() => {
+      searchSourceStub = createSearchSourceStub([{ _id: 'hit1' }]);
+      fetchAnchor = fetchAnchorProvider(createIndexPatternsStub(), searchSourceStub, true);
+    });
+
+    it('should request fields if useNewFieldsApi set', function () {
+      searchSourceStub._stubHits = [{ property1: 'value1' }, { property2: 'value2' }];
+
+      return fetchAnchor('INDEX_PATTERN_ID', 'id', [
+        { '@timestamp': 'desc' },
+        { _doc: 'desc' },
+      ]).then(() => {
+        const setFieldsSpy = searchSourceStub.setField.withArgs('fields');
+        const removeFieldsSpy = searchSourceStub.removeField.withArgs('fieldsFromSource');
+        expect(setFieldsSpy.calledOnce).toBe(true);
+        expect(removeFieldsSpy.calledOnce).toBe(true);
+        expect(setFieldsSpy.firstCall.args[1]).toEqual(['*']);
+      });
+    });
+  });
 });

src/plugins/discover/public/application/angular/context/api/context.predecessors.test.js

@@ -227,4 +227,81 @@ describe('context app', function () {
       });
     });
   });
+
+  describe('function fetchPredecessors with useNewFieldsApi set', function () {
+    let fetchPredecessors;
+    let mockSearchSource;
+
+    beforeEach(() => {
+      mockSearchSource = createContextSearchSourceStub([], '@timestamp', MS_PER_DAY * 8);
+
+      setServices({
+        data: {
+          search: {
+            searchSource: {
+              create: jest.fn().mockImplementation(() => mockSearchSource),
+            },
+          },
+        },
+      });
+
+      fetchPredecessors = (
+        indexPatternId,
+        timeField,
+        sortDir,
+        timeValIso,
+        timeValNr,
+        tieBreakerField,
+        tieBreakerValue,
+        size
+      ) => {
+        const anchor = {
+          _source: {
+            [timeField]: timeValIso,
+          },
+          sort: [timeValNr, tieBreakerValue],
+        };
+
+        return fetchContextProvider(createIndexPatternsStub(), true).fetchSurroundingDocs(
+          'predecessors',
+          indexPatternId,
+          anchor,
+          timeField,
+          tieBreakerField,
+          sortDir,
+          size,
+          []
+        );
+      };
+    });
+
+    it('should perform exactly one query when enough hits are returned', function () {
+      mockSearchSource._stubHits = [
+        mockSearchSource._createStubHit(MS_PER_DAY * 3000 + 2),
+        mockSearchSource._createStubHit(MS_PER_DAY * 3000 + 1),
+        mockSearchSource._createStubHit(MS_PER_DAY * 3000),
+        mockSearchSource._createStubHit(MS_PER_DAY * 2000),
+        mockSearchSource._createStubHit(MS_PER_DAY * 1000),
+      ];
+
+      return fetchPredecessors(
+        'INDEX_PATTERN_ID',
+        '@timestamp',
+        'desc',
+        ANCHOR_TIMESTAMP_3000,
+        MS_PER_DAY * 3000,
+        '_doc',
+        0,
+        3,
+        []
+      ).then((hits) => {
+        const setFieldsSpy = mockSearchSource.setField.withArgs('fields');
+        const removeFieldsSpy = mockSearchSource.removeField.withArgs('fieldsFromSource');
+        expect(mockSearchSource.fetch.calledOnce).toBe(true);
+        expect(removeFieldsSpy.calledOnce).toBe(true);
+        expect(setFieldsSpy.calledOnce).toBe(true);
+        expect(hits).toEqual(mockSearchSource._stubHits.slice(0, 3));
+      });
+    });
+  });
 });

src/plugins/discover/public/application/angular/context/api/context.successors.test.js

@@ -231,4 +231,81 @@ describe('context app', function () {
       });
     });
   });
+
+  describe('function fetchSuccessors with useNewFieldsApi set', function () {
+    let fetchSuccessors;
+    let mockSearchSource;
+
+    beforeEach(() => {
+      mockSearchSource = createContextSearchSourceStub([], '@timestamp');
+
+      setServices({
+        data: {
+          search: {
+            searchSource: {
+              create: jest.fn().mockImplementation(() => mockSearchSource),
+            },
+          },
+        },
+      });
+
+      fetchSuccessors = (
+        indexPatternId,
+        timeField,
+        sortDir,
+        timeValIso,
+        timeValNr,
+        tieBreakerField,
+        tieBreakerValue,
+        size
+      ) => {
+        const anchor = {
+          _source: {
+            [timeField]: timeValIso,
+          },
+          sort: [timeValNr, tieBreakerValue],
+        };
+
+        return fetchContextProvider(createIndexPatternsStub(), true).fetchSurroundingDocs(
+          'successors',
+          indexPatternId,
+          anchor,
+          timeField,
+          tieBreakerField,
+          sortDir,
+          size,
+          []
+        );
+      };
+    });
+
+    it('should perform exactly one query when enough hits are returned', function () {
+      mockSearchSource._stubHits = [
+        mockSearchSource._createStubHit(MS_PER_DAY * 5000),
+        mockSearchSource._createStubHit(MS_PER_DAY * 4000),
+        mockSearchSource._createStubHit(MS_PER_DAY * 3000),
+        mockSearchSource._createStubHit(MS_PER_DAY * 3000 - 1),
+        mockSearchSource._createStubHit(MS_PER_DAY * 3000 - 2),
+      ];
+
+      return fetchSuccessors(
+        'INDEX_PATTERN_ID',
+        '@timestamp',
+        'desc',
+        ANCHOR_TIMESTAMP_3000,
+        MS_PER_DAY * 3000,
+        '_doc',
+        0,
+        3,
+        []
+      ).then((hits) => {
+        expect(mockSearchSource.fetch.calledOnce).toBe(true);
+        expect(hits).toEqual(mockSearchSource._stubHits.slice(-3));
+        const setFieldsSpy = mockSearchSource.setField.withArgs('fields');
+        const removeFieldsSpy = mockSearchSource.removeField.withArgs('fieldsFromSource');
+        expect(removeFieldsSpy.calledOnce).toBe(true);
+        expect(setFieldsSpy.calledOnce).toBe(true);
+      });
+    });
+  });
 });

src/plugins/discover/public/application/angular/context/api/context.ts

@@ -40,7 +40,7 @@ const DAY_MILLIS = 24 * 60 * 60 * 1000;
 // look from 1 day up to 10000 days into the past and future
 const LOOKUP_OFFSETS = [0, 1, 7, 30, 365, 10000].map((days) => days * DAY_MILLIS);
 
-function fetchContextProvider(indexPatterns: IndexPatternsContract) {
+function fetchContextProvider(indexPatterns: IndexPatternsContract, useNewFieldsApi?: boolean) {
   return {
     fetchSurroundingDocs,
   };
@@ -89,7 +89,14 @@ function fetchContextProvider(indexPatterns: IndexPatternsContract) {
         break;
       }
 
-      const searchAfter = getEsQuerySearchAfter(type, documents, timeField, anchor, nanos);
+      const searchAfter = getEsQuerySearchAfter(
+        type,
+        documents,
+        timeField,
+        anchor,
+        nanos,
+        useNewFieldsApi
+      );
 
       const sort = getEsQuerySort(timeField, tieBreakerField, sortDirToApply);
 
@@ -116,6 +123,10 @@ function fetchContextProvider(indexPatterns: IndexPatternsContract) {
     const { data } = getServices();
 
     const searchSource = await data.search.searchSource.create();
+    if (useNewFieldsApi) {
+      searchSource.removeField('fieldsFromSource');
+      searchSource.setField('fields', ['*']);
+    }
     return searchSource
       .setParent(undefined)
       .setField('index', indexPattern)

src/plugins/discover/public/application/angular/context/api/utils/get_es_query_search_after.ts

@@ -31,16 +31,30 @@ export function getEsQuerySearchAfter(
   documents: EsHitRecordList,
   timeFieldName: string,
   anchor: EsHitRecord,
-  nanoSeconds: string
+  nanoSeconds: string,
+  useNewFieldsApi?: boolean
 ): EsQuerySearchAfter {
   if (documents.length) {
     // already surrounding docs -> first or last record  is used
     const afterTimeRecIdx = type === 'successors' && documents.length ? documents.length - 1 : 0;
     const afterTimeDoc = documents[afterTimeRecIdx];
-    const afterTimeValue = nanoSeconds ? afterTimeDoc._source[timeFieldName] : afterTimeDoc.sort[0];
+    let afterTimeValue = afterTimeDoc.sort[0];
+    if (nanoSeconds) {
+      afterTimeValue = useNewFieldsApi
+        ? afterTimeDoc.fields[timeFieldName][0]
+        : afterTimeDoc._source[timeFieldName];
+    }
     return [afterTimeValue, afterTimeDoc.sort[1]];
   }
   // if data_nanos adapt timestamp value for sorting, since numeric value was rounded by browser
   // ES search_after also works when number is provided as string
-  return [nanoSeconds ? anchor._source[timeFieldName] : anchor.sort[0], anchor.sort[1]];
+  const searchAfter = new Array(2) as EsQuerySearchAfter;
+  searchAfter[0] = anchor.sort[0];
+  if (nanoSeconds) {
+    searchAfter[0] = useNewFieldsApi
+      ? anchor.fields[timeFieldName][0]
+      : anchor._source[timeFieldName];
+  }
+  searchAfter[1] = anchor.sort[1];
+  return searchAfter;
 }

src/plugins/discover/public/application/angular/context/query/actions.js

@@ -27,11 +27,17 @@ import { fetchContextProvider } from '../api/context';
 import { getQueryParameterActions } from '../query_parameters';
 import { FAILURE_REASONS, LOADING_STATUS } from './index';
 import { MarkdownSimple } from '../../../../../../kibana_react/public';
+import { SEARCH_FIELDS_FROM_SOURCE } from '../../../../../common';
 
 export function QueryActionsProvider(Promise) {
-  const { filterManager, indexPatterns, data } = getServices();
-  const fetchAnchor = fetchAnchorProvider(indexPatterns, data.search.searchSource.createEmpty());
-  const { fetchSurroundingDocs } = fetchContextProvider(indexPatterns);
+  const { filterManager, indexPatterns, data, uiSettings } = getServices();
+  const useNewFieldsApi = !uiSettings.get(SEARCH_FIELDS_FROM_SOURCE);
+  const fetchAnchor = fetchAnchorProvider(
+    indexPatterns,
+    data.search.searchSource.createEmpty(),
+    useNewFieldsApi
+  );
+  const { fetchSurroundingDocs } = fetchContextProvider(indexPatterns, useNewFieldsApi);
   const { setPredecessorCount, setQueryParameters, setSuccessorCount } = getQueryParameterActions(
     filterManager,
     indexPatterns

src/plugins/discover/public/application/angular/context_app.html

@@ -17,5 +17,6 @@
   successor-available="contextApp.state.rows.successors.length"
   successor-status="contextApp.state.loadingStatus.successors.status"
   on-change-successor-count="contextApp.actions.fetchGivenSuccessorRows"
+  use-new-fields-api="contextApp.state.useNewFieldsApi"
   top-nav-menu="contextApp.topNavMenu"
 ></context-app-legacy>

src/plugins/discover/public/application/angular/context_app.js

@@ -18,7 +18,11 @@
  */
 
 import _ from 'lodash';
-import { CONTEXT_STEP_SETTING, CONTEXT_TIE_BREAKER_FIELDS_SETTING } from '../../../common';
+import {
+  CONTEXT_STEP_SETTING,
+  CONTEXT_TIE_BREAKER_FIELDS_SETTING,
+  SEARCH_FIELDS_FROM_SOURCE,
+} from '../../../common';
 import { getAngularModule, getServices } from '../../kibana_services';
 import contextAppTemplate from './context_app.html';
 import './context/components/action_bar';
@@ -59,9 +63,11 @@ function ContextAppController($scope, Private) {
   const { filterManager, indexPatterns, uiSettings, navigation } = getServices();
   const queryParameterActions = getQueryParameterActions(filterManager, indexPatterns);
   const queryActions = Private(QueryActionsProvider);
+  const useNewFieldsApi = !uiSettings.get(SEARCH_FIELDS_FROM_SOURCE);
   this.state = createInitialState(
     parseInt(uiSettings.get(CONTEXT_STEP_SETTING), 10),
-    getFirstSortableField(this.indexPattern, uiSettings.get(CONTEXT_TIE_BREAKER_FIELDS_SETTING))
+    getFirstSortableField(this.indexPattern, uiSettings.get(CONTEXT_TIE_BREAKER_FIELDS_SETTING)),
+    useNewFieldsApi
   );
   this.topNavMenu = navigation.ui.TopNavMenu;
 
@@ -127,7 +133,7 @@ function ContextAppController($scope, Private) {
   );
 }
 
-function createInitialState(defaultStepSize, tieBreakerField) {
+function createInitialState(defaultStepSize, tieBreakerField, useNewFieldsApi) {
   return {
     queryParameters: createInitialQueryParametersState(defaultStepSize, tieBreakerField),
     rows: {
@@ -137,5 +143,6 @@ function createInitialState(defaultStepSize, tieBreakerField) {
       successors: [],
     },
     loadingStatus: createInitialLoadingStatusState(),
+    useNewFieldsApi,
   };
 }