From 9c1202f16ac103101d7529747d7bfeadd0afc557 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Wed, 22 Jun 2022 18:27:34 +0200 Subject: [PATCH] [Discover] Centralize document flattening and introduce DataTableRecord (#134174) --- .../discover/public/__fixtures__/fake_row.js | 25 ---- .../{real_hits.js => real_hits.ts} | 31 ++--- .../discover/public/__mocks__/grid_context.ts | 18 ++- .../__mocks__/use_context_app_fetch.tsx | 26 ++-- .../application/context/context_app.tsx | 4 +- .../context/context_app_content.test.tsx | 4 +- .../context/context_app_content.tsx | 13 +- .../context/hooks/use_context_app_fetch.tsx | 8 +- .../application/context/services/_stubs.ts | 4 +- .../context/services/anchor.test.ts | 15 +- .../application/context/services/anchor.ts | 11 +- .../services/context.predecessors.test.ts | 78 +++++++---- .../services/context.successors.test.ts | 62 ++++++--- .../application/context/services/context.ts | 19 +-- .../context/services/context_query_state.ts | 10 +- .../context/utils/fetch_hits_in_interval.ts | 12 +- .../utils/get_es_query_search_after.ts | 27 ++-- .../layout/discover_documents.test.tsx | 11 +- .../components/layout/discover_documents.tsx | 6 +- .../layout/discover_layout.test.tsx | 4 +- .../main/components/layout/types.ts | 6 +- .../sidebar/discover_sidebar.test.tsx | 14 +- .../components/sidebar/discover_sidebar.tsx | 5 +- .../discover_sidebar_responsive.test.tsx | 14 +- .../sidebar/discover_sidebar_responsive.tsx | 7 +- .../sidebar/lib/field_calculator.js | 7 +- .../sidebar/lib/field_calculator.test.ts | 12 +- .../components/sidebar/lib/get_details.ts | 5 +- .../application/main/discover_main_app.tsx | 4 +- .../main/hooks/use_discover_state.ts | 4 +- .../main/hooks/use_saved_search.ts | 7 +- .../main/utils/calc_field_counts.test.ts | 18 +-- .../main/utils/calc_field_counts.ts | 7 +- .../application/main/utils/fetch_all.test.ts | 7 +- .../application/main/utils/fetch_all.ts | 10 +- .../application/main/utils/fetch_chart.ts | 9 +- .../application/main/utils/fetch_documents.ts | 2 +- .../discover/public/application/types.ts | 11 -- .../discover_grid/discover_grid.test.tsx | 11 +- .../discover_grid/discover_grid.tsx | 35 ++--- .../discover_grid_cell_actions.tsx | 4 +- .../discover_grid/discover_grid_context.tsx | 9 +- .../discover_grid_document_selection.test.tsx | 7 +- .../discover_grid_document_selection.tsx | 39 +++--- .../discover_grid_expand_button.tsx | 12 +- .../discover_grid_flyout.test.tsx | 28 ++-- .../discover_grid/discover_grid_flyout.tsx | 30 ++-- .../get_render_cell_value.test.tsx | 64 ++++----- .../discover_grid/get_render_cell_value.tsx | 55 ++++---- .../doc_table/components/table_row.test.tsx | 6 +- .../doc_table/components/table_row.tsx | 29 ++-- .../doc_table/doc_table_wrapper.test.tsx | 34 +++-- .../doc_table/doc_table_wrapper.tsx | 13 +- .../doc_table/utils/row_formatter.test.ts | 41 +++--- .../doc_table/utils/row_formatter.tsx | 6 +- .../embeddable/saved_search_embeddable.tsx | 10 +- .../public/embeddable/saved_search_grid.tsx | 4 +- .../public/hooks/use_es_doc_search.test.tsx | 8 +- .../public/hooks/use_es_doc_search.ts | 9 +- src/plugins/discover/public/plugin.tsx | 4 +- .../components/doc_viewer/doc_viewer.test.tsx | 5 +- .../doc_viewer/doc_viewer_tab.test.tsx | 7 +- .../components/doc_viewer/doc_viewer_tab.tsx | 5 +- .../__snapshots__/source.test.tsx.snap | 128 ------------------ .../doc_viewer_source/source.test.tsx | 9 +- .../components/doc_viewer_source/source.tsx | 2 +- .../doc_viewer_table/legacy/table.test.tsx | 106 ++++++++------- .../doc_viewer_table/legacy/table.tsx | 14 +- .../components/doc_viewer_table/table.tsx | 9 +- .../services/doc_views/doc_views_registry.ts | 4 +- .../services/doc_views/doc_views_types.ts | 6 +- src/plugins/discover/public/types.ts | 31 ++++- .../public/utils/build_data_record.ts | 43 ++++++ .../utils/convert_value_to_string.test.tsx | 29 ---- .../public/utils/convert_value_to_string.ts | 16 ++- .../utils/copy_value_to_clipboard.test.tsx | 1 - .../discover/public/utils/format_hit.test.ts | 25 +++- .../discover/public/utils/format_hit.ts | 14 +- .../discover/public/utils/get_doc_id.tsx | 17 +++ 79 files changed, 667 insertions(+), 799 deletions(-) delete mode 100644 src/plugins/discover/public/__fixtures__/fake_row.js rename src/plugins/discover/public/__fixtures__/{real_hits.js => real_hits.ts} (89%) delete mode 100644 src/plugins/discover/public/services/doc_views/components/doc_viewer_source/__snapshots__/source.test.tsx.snap create mode 100644 src/plugins/discover/public/utils/build_data_record.ts create mode 100644 src/plugins/discover/public/utils/get_doc_id.tsx diff --git a/src/plugins/discover/public/__fixtures__/fake_row.js b/src/plugins/discover/public/__fixtures__/fake_row.js deleted file mode 100644 index 8cba3d85a69c3..0000000000000 --- a/src/plugins/discover/public/__fixtures__/fake_row.js +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -const longString = Array(200).join('_'); - -export function getFakeRowVals(type, id, mapping) { - return mapping.reduce((collector, field) => { - collector[field.name] = `${field.name}_${type}_${id}_${longString}`; - return collector; - }, {}); -} - -export function getFakeRow(id, mapping) { - return { - _id: id, - _index: 'test', - _source: getFakeRowVals('original', id, mapping), - sort: [id], - }; -} diff --git a/src/plugins/discover/public/__fixtures__/real_hits.js b/src/plugins/discover/public/__fixtures__/real_hits.ts similarity index 89% rename from src/plugins/discover/public/__fixtures__/real_hits.js rename to src/plugins/discover/public/__fixtures__/real_hits.ts index 0286d036de090..96d084a977bd4 100644 --- a/src/plugins/discover/public/__fixtures__/real_hits.js +++ b/src/plugins/discover/public/__fixtures__/real_hits.ts @@ -5,7 +5,10 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ - +import type { DataView } from '@kbn/data-views-plugin/common'; +import { cloneDeep } from 'lodash'; +import { buildDataTableRecord } from '../utils/build_data_record'; +import type { EsHitRecord } from '../types'; /* Extensions: gif: 5 @@ -23,10 +26,9 @@ All have the same index, ids are unique */ -export default [ +export const realHits: EsHitRecord[] = [ { _index: 'logstash-2014.09.09', - _type: 'apache', _id: '61', _score: 1, _source: { @@ -36,7 +38,6 @@ export default [ }, { _index: 'logstash-2014.09.09', - _type: 'apache', _id: '388', _score: 1, _source: { @@ -46,7 +47,6 @@ export default [ }, { _index: 'logstash-2014.09.09', - _type: 'apache', _id: '403', _score: 1, _source: { @@ -56,7 +56,6 @@ export default [ }, { _index: 'logstash-2014.09.09', - _type: 'apache', _id: '415', _score: 1, _source: { @@ -66,7 +65,6 @@ export default [ }, { _index: 'logstash-2014.09.09', - _type: 'apache', _id: '460', _score: 1, _source: { @@ -77,7 +75,6 @@ export default [ }, { _index: 'logstash-2014.09.09', - _type: 'apache', _id: '496', _score: 1, _source: { @@ -87,7 +84,6 @@ export default [ }, { _index: 'logstash-2014.09.09', - _type: 'apache', _id: '511', _score: 1, _source: { @@ -97,7 +93,6 @@ export default [ }, { _index: 'logstash-2014.09.09', - _type: 'apache', _id: '701', _score: 1, _source: { @@ -107,7 +102,6 @@ export default [ }, { _index: 'logstash-2014.09.09', - _type: 'apache', _id: '838', _score: 1, _source: { @@ -118,7 +112,6 @@ export default [ }, { _index: 'logstash-2014.09.09', - _type: 'apache', _id: '890', _score: 1, _source: { @@ -129,7 +122,6 @@ export default [ }, { _index: 'logstash-2014.09.09', - _type: 'nginx', _id: '927', _score: 1, _source: { @@ -140,7 +132,6 @@ export default [ }, { _index: 'logstash-2014.09.09', - _type: 'apache', _id: '1034', _score: 1, _source: { @@ -150,7 +141,6 @@ export default [ }, { _index: 'logstash-2014.09.09', - _type: 'apache', _id: '1142', _score: 1, _source: { @@ -161,7 +151,6 @@ export default [ }, { _index: 'logstash-2014.09.09', - _type: 'apache', _id: '1180', _score: 1, _source: { @@ -171,7 +160,6 @@ export default [ }, { _index: 'logstash-2014.09.09', - _type: 'nginx', _id: '1224', _score: 1, _source: { @@ -181,7 +169,6 @@ export default [ }, { _index: 'logstash-2014.09.09', - _type: 'apache', _id: '1243', _score: 1, _source: { @@ -191,7 +178,6 @@ export default [ }, { _index: 'logstash-2014.09.09', - _type: 'apache', _id: '1510', _score: 1, _source: { @@ -201,7 +187,6 @@ export default [ }, { _index: 'logstash-2014.09.09', - _type: 'apache', _id: '1628', _score: 1, _source: { @@ -211,7 +196,6 @@ export default [ }, { _index: 'logstash-2014.09.09', - _type: 'apache', _id: '1729', _score: 1, _source: { @@ -221,7 +205,6 @@ export default [ }, { _index: 'logstash-2014.09.09', - _type: 'apache', _id: '1945', _score: 1, _source: { @@ -230,3 +213,7 @@ export default [ }, }, ]; + +export function getDataTableRecords(dataView: DataView) { + return cloneDeep(realHits).map((hit: EsHitRecord) => buildDataTableRecord(hit, dataView)); +} diff --git a/src/plugins/discover/public/__mocks__/grid_context.ts b/src/plugins/discover/public/__mocks__/grid_context.ts index 3f760bc4a4258..7e35aaf42204c 100644 --- a/src/plugins/discover/public/__mocks__/grid_context.ts +++ b/src/plugins/discover/public/__mocks__/grid_context.ts @@ -6,7 +6,6 @@ * Side Public License, v 1. */ -import { flattenHit } from '@kbn/data-plugin/common'; import type { DataView } from '@kbn/data-views-plugin/public'; import { indexPatternMock } from './index_pattern'; import { dataViewComplexMock } from './data_view_complex'; @@ -15,18 +14,18 @@ import { esHitsComplex } from './es_hits_complex'; import { discoverServiceMock } from './services'; import { GridContext } from '../components/discover_grid/discover_grid_context'; import { convertValueToString } from '../utils/convert_value_to_string'; -import type { ElasticSearchHit } from '../types'; +import { buildDataTableRecord } from '../utils/build_data_record'; +import { EsHitRecord } from '../types'; -const buildGridContext = (dataView: DataView, rows: ElasticSearchHit[]): GridContext => { - const rowsFlattened = rows.map((hit) => - flattenHit(hit, dataView, { includeIgnoredValues: true }) - ); +const buildGridContext = (dataView: DataView, rows: EsHitRecord[]): GridContext => { + const usedRows = rows.map((row) => { + return buildDataTableRecord(row, dataView); + }); return { expanded: undefined, setExpanded: jest.fn(), - rows, - rowsFlattened, + rows: usedRows, onFilter: jest.fn(), indexPattern: dataView, isDarkMode: false, @@ -37,8 +36,7 @@ const buildGridContext = (dataView: DataView, rows: ElasticSearchHit[]): GridCon rowIndex, columnId, services: discoverServiceMock, - rows, - rowsFlattened, + rows: usedRows, dataView, options, }), diff --git a/src/plugins/discover/public/application/context/__mocks__/use_context_app_fetch.tsx b/src/plugins/discover/public/application/context/__mocks__/use_context_app_fetch.tsx index 1499067d2332e..be424f12d93f3 100644 --- a/src/plugins/discover/public/application/context/__mocks__/use_context_app_fetch.tsx +++ b/src/plugins/discover/public/application/context/__mocks__/use_context_app_fetch.tsx @@ -6,14 +6,20 @@ * Side Public License, v 1. */ -export const mockAnchorHit = { - _id: '123', - _index: 'the-index-pattern-id', - fields: { order_date: ['2021-06-07T18:52:17.000Z'] }, - sort: [1623091937000, 2092], - isAnchor: true, - _version: 1, -}; +import { indexPatternMock } from '../../../__mocks__/index_pattern'; +import { buildDataTableRecord } from '../../../utils/build_data_record'; + +export const mockAnchorHit = buildDataTableRecord( + { + _id: '123', + _index: 'the-index-pattern-id', + fields: { order_date: ['2021-06-07T18:52:17.000Z'] }, + sort: [1623091937000, 2092], + _version: 1, + }, + indexPatternMock, + true +); export const mockPredecessorHits = [ { @@ -37,7 +43,7 @@ export const mockPredecessorHits = [ sort: ['2021-06-07T19:10:22.000Z', 2435], _version: 1, }, -]; +].map((entry) => buildDataTableRecord(entry, indexPatternMock)); export const mockSuccessorHits = [ { @@ -61,4 +67,4 @@ export const mockSuccessorHits = [ sort: ['2021-06-07T18:47:16.000Z', 2437], _version: 1, }, -]; +].map((entry) => buildDataTableRecord(entry, indexPatternMock)); diff --git a/src/plugins/discover/public/application/context/context_app.tsx b/src/plugins/discover/public/application/context/context_app.tsx index b56364781206b..47650b64ae8cf 100644 --- a/src/plugins/discover/public/application/context/context_app.tsx +++ b/src/plugins/discover/public/application/context/context_app.tsx @@ -95,7 +95,7 @@ export const ContextApp = ({ indexPattern, anchorId }: ContextAppProps) => { fetchContextRows, fetchAllRows, fetchSurroundingRows, - fetchedState.anchor._id, + fetchedState.anchor.id, ]); const { columns, onAddColumn, onRemoveColumn, onSetColumns } = useColumns({ @@ -110,7 +110,7 @@ export const ContextApp = ({ indexPattern, anchorId }: ContextAppProps) => { const rows = useMemo( () => [ ...(fetchedState.predecessors || []), - ...(fetchedState.anchor._id ? [fetchedState.anchor] : []), + ...(fetchedState.anchor.id ? [fetchedState.anchor] : []), ...(fetchedState.successors || []), ], [fetchedState.predecessors, fetchedState.anchor, fetchedState.successors] diff --git a/src/plugins/discover/public/application/context/context_app_content.test.tsx b/src/plugins/discover/public/application/context/context_app_content.test.tsx index 9751f0d38f3d0..23a475c38b940 100644 --- a/src/plugins/discover/public/application/context/context_app_content.test.tsx +++ b/src/plugins/discover/public/application/context/context_app_content.test.tsx @@ -18,8 +18,8 @@ import { indexPatternMock } from '../../__mocks__/index_pattern'; import { DiscoverGrid } from '../../components/discover_grid/discover_grid'; import { discoverServiceMock } from '../../__mocks__/services'; import { DocTableWrapper } from '../../components/doc_table/doc_table_wrapper'; -import { EsHitRecordList } from '../types'; import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; +import { buildDataTableRecord } from '../../utils/build_data_record'; describe('ContextAppContent test', () => { const mountComponent = ({ @@ -56,7 +56,7 @@ describe('ContextAppContent test', () => { anchorStatus: anchorStatus || LoadingStatus.LOADED, predecessorsStatus: LoadingStatus.LOADED, successorsStatus: LoadingStatus.LOADED, - rows: [hit] as unknown as EsHitRecordList, + rows: [buildDataTableRecord(hit, indexPatternMock)], predecessors: [], successors: [], defaultStepSize: 5, diff --git a/src/plugins/discover/public/application/context/context_app_content.tsx b/src/plugins/discover/public/application/context/context_app_content.tsx index 0c14c3bd62e34..ce9dd8af939d3 100644 --- a/src/plugins/discover/public/application/context/context_app_content.tsx +++ b/src/plugins/discover/public/application/context/context_app_content.tsx @@ -20,10 +20,9 @@ import { AppState } from './services/context_state'; import { SurrDocType } from './services/context'; import { MAX_CONTEXT_SIZE, MIN_CONTEXT_SIZE } from './services/constants'; import { DocTableContext } from '../../components/doc_table/doc_table_context'; -import { EsHitRecordList } from '../types'; -import { SortPairArr } from '../../components/doc_table/utils/get_sort'; -import { ElasticSearchHit } from '../../types'; +import type { SortPairArr } from '../../components/doc_table/utils/get_sort'; import { useDiscoverServices } from '../../hooks/use_discover_services'; +import type { DataTableRecord } from '../../types'; export interface ContextAppContentProps { columns: string[]; @@ -33,9 +32,9 @@ export interface ContextAppContentProps { indexPattern: DataView; predecessorCount: number; successorCount: number; - rows: EsHitRecordList; - predecessors: EsHitRecordList; - successors: EsHitRecordList; + rows: DataTableRecord[]; + predecessors: DataTableRecord[]; + successors: DataTableRecord[]; anchorStatus: LoadingStatus; predecessorsStatus: LoadingStatus; successorsStatus: LoadingStatus; @@ -76,7 +75,7 @@ export function ContextAppContent({ }: ContextAppContentProps) { const { uiSettings: config } = useDiscoverServices(); - const [expandedDoc, setExpandedDoc] = useState(); + const [expandedDoc, setExpandedDoc] = useState(); const isAnchorLoading = anchorStatus === LoadingStatus.LOADING || anchorStatus === LoadingStatus.UNINITIALIZED; const arePredecessorsLoading = diff --git a/src/plugins/discover/public/application/context/hooks/use_context_app_fetch.tsx b/src/plugins/discover/public/application/context/hooks/use_context_app_fetch.tsx index a43b8b0ca4f9d..1201526da0821 100644 --- a/src/plugins/discover/public/application/context/hooks/use_context_app_fetch.tsx +++ b/src/plugins/discover/public/application/context/hooks/use_context_app_fetch.tsx @@ -21,8 +21,8 @@ import { } from '../services/context_query_state'; import { AppState } from '../services/context_state'; import { getFirstSortableField } from '../utils/sorting'; -import { EsHitRecord } from '../../types'; import { useDiscoverServices } from '../../../hooks/use_discover_services'; +import type { DataTableRecord } from '../../../types'; const createError = (statusKey: string, reason: FailureReason, error?: Error) => ({ [statusKey]: { value: LoadingStatus.FAILED, error, reason }, @@ -117,7 +117,7 @@ export function useContextAppFetch({ ]); const fetchSurroundingRows = useCallback( - async (type: SurrDocType, fetchedAnchor?: EsHitRecord) => { + async (type: SurrDocType, fetchedAnchor?: DataTableRecord) => { const filters = filterManager.getFilters(); const count = @@ -133,7 +133,7 @@ export function useContextAppFetch({ const rows = await fetchSurroundingDocs( type, indexPattern, - anchor as EsHitRecord, + anchor, tieBreakerField, SortDirection.desc, count, @@ -167,7 +167,7 @@ export function useContextAppFetch({ ); const fetchContextRows = useCallback( - (anchor?: EsHitRecord) => + (anchor?: DataTableRecord) => Promise.allSettled([ fetchSurroundingRows(SurrDocType.PREDECESSORS, anchor), fetchSurroundingRows(SurrDocType.SUCCESSORS, anchor), diff --git a/src/plugins/discover/public/application/context/services/_stubs.ts b/src/plugins/discover/public/application/context/services/_stubs.ts index b37c1ecf4efbf..f2375372829bc 100644 --- a/src/plugins/discover/public/application/context/services/_stubs.ts +++ b/src/plugins/discover/public/application/context/services/_stubs.ts @@ -11,7 +11,7 @@ import moment from 'moment'; import { of } from 'rxjs'; import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { IKibanaSearchResponse } from '@kbn/data-plugin/common'; -import { EsHitRecordList } from '../../types'; +import { EsHitRecord } from '../../../types'; type SortHit = { [key in string]: number; // timeField name @@ -22,7 +22,7 @@ type SortHit = { /** * A stubbed search source with a `fetch` method that returns all of `_stubHits`. */ -export function createSearchSourceStub(hits: EsHitRecordList, timeField?: string) { +export function createSearchSourceStub(hits: EsHitRecord[], timeField?: string) { const requestResult = { id: 'Fjk5bndxTHJWU2FldVRVQ0tYR0VqOFEcRWtWNDhOdG5SUzJYcFhONVVZVTBJQToxMDMwOQ==', rawResponse: { diff --git a/src/plugins/discover/public/application/context/services/anchor.test.ts b/src/plugins/discover/public/application/context/services/anchor.test.ts index ab613fb71cc6d..0fff0193e4995 100644 --- a/src/plugins/discover/public/application/context/services/anchor.test.ts +++ b/src/plugins/discover/public/application/context/services/anchor.test.ts @@ -11,7 +11,6 @@ import { createSearchSourceStub } from './_stubs'; import { fetchAnchor, updateSearchSource } from './anchor'; import { indexPatternMock } from '../../../__mocks__/index_pattern'; import { savedSearchMock } from '../../../__mocks__/saved_search'; -import { EsHitRecordList } from '../../types'; describe('context app', function () { // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -24,7 +23,7 @@ describe('context app', function () { describe('function fetchAnchor', function () { beforeEach(() => { - searchSourceStub = createSearchSourceStub([{ _id: 'hit1' }] as unknown as EsHitRecordList); + searchSourceStub = createSearchSourceStub([{ _id: 'hit1', _index: 'test' }]); }); it('should use the `fetch$` method of the SearchSource', function () { @@ -142,7 +141,7 @@ describe('context app', function () { }); it('should reject with an error when no hits were found', function () { - searchSourceStub = createSearchSourceStub([] as unknown as EsHitRecordList); + searchSourceStub = createSearchSourceStub([]); return fetchAnchor('id', indexPattern, searchSourceStub, [ { '@timestamp': SortDirection.desc }, @@ -159,15 +158,15 @@ describe('context app', function () { it('should return the first hit after adding an anchor marker', function () { searchSourceStub = createSearchSourceStub([ - { property1: 'value1' }, - { property2: 'value2' }, - ] as unknown as EsHitRecordList); + { _id: '1', _index: 't' }, + { _id: '3', _index: 't' }, + ]); return fetchAnchor('id', indexPattern, searchSourceStub, [ { '@timestamp': SortDirection.desc }, { _doc: SortDirection.desc }, ]).then((anchorDocument) => { - expect(anchorDocument).toHaveProperty('property1', 'value1'); + expect(anchorDocument).toHaveProperty('raw._id', '1'); expect(anchorDocument).toHaveProperty('isAnchor', true); }); }); @@ -175,7 +174,7 @@ describe('context app', function () { describe('useNewFields API', () => { beforeEach(() => { - searchSourceStub = createSearchSourceStub([{ _id: 'hit1' }] as unknown as EsHitRecordList); + searchSourceStub = createSearchSourceStub([{ _id: 'hit1', _index: 't' }]); }); it('should request fields if useNewFieldsApi set', function () { diff --git a/src/plugins/discover/public/application/context/services/anchor.ts b/src/plugins/discover/public/application/context/services/anchor.ts index 28d2298513aa7..77644c8af5808 100644 --- a/src/plugins/discover/public/application/context/services/anchor.ts +++ b/src/plugins/discover/public/application/context/services/anchor.ts @@ -9,7 +9,8 @@ import { lastValueFrom } from 'rxjs'; import { i18n } from '@kbn/i18n'; import { ISearchSource, EsQuerySortValue } from '@kbn/data-plugin/public'; import { DataView } from '@kbn/data-views-plugin/public'; -import { EsHitRecord } from '../../types'; +import { DataTableRecord } from '../../../types'; +import { buildDataTableRecord } from '../../../utils/build_data_record'; export async function fetchAnchor( anchorId: string, @@ -17,7 +18,7 @@ export async function fetchAnchor( searchSource: ISearchSource, sort: EsQuerySortValue[], useNewFieldsApi: boolean = false -): Promise { +): Promise { updateSearchSource(searchSource, anchorId, sort, useNewFieldsApi, indexPattern); const { rawResponse } = await lastValueFrom(await searchSource.fetch$()); const doc = rawResponse.hits?.hits?.[0]; @@ -29,11 +30,7 @@ export async function fetchAnchor( }) ); } - - return { - ...doc, - isAnchor: true, - } as EsHitRecord; + return buildDataTableRecord(doc, indexPattern, true); } export function updateSearchSource( diff --git a/src/plugins/discover/public/application/context/services/context.predecessors.test.ts b/src/plugins/discover/public/application/context/services/context.predecessors.test.ts index d9b55303ad507..42d53dc66a878 100644 --- a/src/plugins/discover/public/application/context/services/context.predecessors.test.ts +++ b/src/plugins/discover/public/application/context/services/context.predecessors.test.ts @@ -14,7 +14,8 @@ import { Query } from '@kbn/es-query'; import { createContextSearchSourceStub } from './_stubs'; import { fetchSurroundingDocs, SurrDocType } from './context'; import { DataPublicPluginStart } from '@kbn/data-plugin/public'; -import { EsHitRecord, EsHitRecordList } from '../../types'; +import { DataTableRecord, EsHitRecord } from '../../../types'; +import { buildDataTableRecord, buildDataTableRecordList } from '../../../utils/build_data_record'; const MS_PER_DAY = 24 * 60 * 60 * 1000; const ANCHOR_TIMESTAMP = new Date(MS_PER_DAY).toJSON(); @@ -36,7 +37,7 @@ describe('context predecessors', function () { tieBreakerField: string, tieBreakerValue: number, size: number - ) => Promise; + ) => Promise; // eslint-disable-next-line @typescript-eslint/no-explicit-any let mockSearchSource: any; @@ -45,6 +46,9 @@ describe('context predecessors', function () { timeFieldName: '@timestamp', isTimeNanosBased: () => false, popularizeField: () => {}, + fields: { + getByName: jest.fn(), + }, } as unknown as DataView; describe('function fetchPredecessors', function () { @@ -59,17 +63,21 @@ describe('context predecessors', function () { } as unknown as DataPublicPluginStart; fetchPredecessors = (timeValIso, timeValNr, tieBreakerField, tieBreakerValue, size = 10) => { - const anchor = { - _source: { - [indexPattern.timeFieldName!]: timeValIso, - }, - sort: [timeValNr, tieBreakerValue], - }; + const anchor = buildDataTableRecord( + { + _source: { + [indexPattern.timeFieldName!]: timeValIso, + }, + sort: [timeValNr, tieBreakerValue], + } as EsHitRecord, + indexPattern, + true + ); return fetchSurroundingDocs( SurrDocType.PREDECESSORS, indexPattern, - anchor as EsHitRecord, + anchor, tieBreakerField, SortDirection.desc, size, @@ -89,9 +97,11 @@ describe('context predecessors', function () { ]; return fetchPredecessors(ANCHOR_TIMESTAMP_3000, MS_PER_DAY * 3000, '_doc', 0, 3).then( - (hits: EsHitRecordList) => { + (hits) => { expect(mockSearchSource.fetch$.calledOnce).toBe(true); - expect(hits).toEqual(mockSearchSource._stubHits.slice(0, 3)); + expect(hits).toEqual( + buildDataTableRecordList(mockSearchSource._stubHits.slice(0, 3), indexPattern) + ); } ); }); @@ -106,7 +116,7 @@ describe('context predecessors', function () { ]; return fetchPredecessors(ANCHOR_TIMESTAMP_3000, MS_PER_DAY * 3000, '_doc', 0, 6).then( - (hits: EsHitRecordList) => { + (hits) => { const intervals: Timestamp[] = mockSearchSource.setField.args .filter(([property]: string) => property === 'query') .map(([, { query }]: [string, { query: Query }]) => @@ -121,8 +131,9 @@ describe('context predecessors', function () { // should have ended with a half-open interval expect(Object.keys(last(intervals) ?? {})).toEqual(['format', 'gte']); expect(intervals.length).toBeGreaterThan(1); - - expect(hits).toEqual(mockSearchSource._stubHits.slice(0, 3)); + expect(hits).toEqual( + buildDataTableRecordList(mockSearchSource._stubHits.slice(0, 3), indexPattern) + ); } ); }); @@ -136,7 +147,7 @@ describe('context predecessors', function () { ]; return fetchPredecessors(ANCHOR_TIMESTAMP_1000, MS_PER_DAY * 1000, '_doc', 0, 3).then( - (hits: EsHitRecordList) => { + (hits) => { const intervals: Timestamp[] = mockSearchSource.setField.args .filter(([property]: string) => property === 'query') .map(([, { query }]: [string, { query: Query }]) => { @@ -155,17 +166,18 @@ describe('context predecessors', function () { // should have stopped before reaching MS_PER_DAY * 1700 expect(moment(last(intervals)?.lte).valueOf()).toBeLessThan(MS_PER_DAY * 1700); expect(intervals.length).toBeGreaterThan(1); - expect(hits).toEqual(mockSearchSource._stubHits.slice(-3)); + + expect(hits).toEqual( + buildDataTableRecordList(mockSearchSource._stubHits.slice(-3), indexPattern) + ); } ); }); it('should return an empty array when no hits were found', function () { - return fetchPredecessors(ANCHOR_TIMESTAMP_3, MS_PER_DAY * 3, '_doc', 0, 3).then( - (hits: EsHitRecordList) => { - expect(hits).toEqual([]); - } - ); + return fetchPredecessors(ANCHOR_TIMESTAMP_3, MS_PER_DAY * 3, '_doc', 0, 3).then((hits) => { + expect(hits).toEqual([]); + }); }); it('should configure the SearchSource to not inherit from the implicit root', function () { @@ -201,17 +213,21 @@ describe('context predecessors', function () { } as unknown as DataPublicPluginStart; fetchPredecessors = (timeValIso, timeValNr, tieBreakerField, tieBreakerValue, size = 10) => { - const anchor = { - _source: { - [indexPattern.timeFieldName!]: timeValIso, - }, - sort: [timeValNr, tieBreakerValue], - }; + const anchor = buildDataTableRecord( + { + _source: { + [indexPattern.timeFieldName!]: timeValIso, + }, + sort: [timeValNr, tieBreakerValue], + } as EsHitRecord, + indexPattern, + true + ); return fetchSurroundingDocs( SurrDocType.PREDECESSORS, indexPattern, - anchor as EsHitRecord, + anchor, tieBreakerField, SortDirection.desc, size, @@ -232,13 +248,15 @@ describe('context predecessors', function () { ]; return fetchPredecessors(ANCHOR_TIMESTAMP_3000, MS_PER_DAY * 3000, '_doc', 0, 3).then( - (hits: EsHitRecordList) => { + (hits) => { const setFieldsSpy = mockSearchSource.setField.withArgs('fields'); const removeFieldsSpy = mockSearchSource.removeField.withArgs('fieldsFromSource'); expect(mockSearchSource.fetch$.calledOnce).toBe(true); expect(removeFieldsSpy.calledOnce).toBe(true); expect(setFieldsSpy.calledOnce).toBe(true); - expect(hits).toEqual(mockSearchSource._stubHits.slice(0, 3)); + expect(hits).toEqual( + buildDataTableRecordList(mockSearchSource._stubHits.slice(0, 3), indexPattern) + ); } ); }); diff --git a/src/plugins/discover/public/application/context/services/context.successors.test.ts b/src/plugins/discover/public/application/context/services/context.successors.test.ts index c9f819c6497d9..dfc57b1859cb1 100644 --- a/src/plugins/discover/public/application/context/services/context.successors.test.ts +++ b/src/plugins/discover/public/application/context/services/context.successors.test.ts @@ -14,7 +14,8 @@ import { createContextSearchSourceStub } from './_stubs'; import { DataPublicPluginStart } from '@kbn/data-plugin/public'; import { Query } from '@kbn/es-query'; import { fetchSurroundingDocs, SurrDocType } from './context'; -import { EsHitRecord, EsHitRecordList } from '../../types'; +import { DataTableRecord } from '../../../types'; +import { buildDataTableRecord, buildDataTableRecordList } from '../../../utils/build_data_record'; const MS_PER_DAY = 24 * 60 * 60 * 1000; const ANCHOR_TIMESTAMP = new Date(MS_PER_DAY).toJSON(); @@ -34,7 +35,7 @@ describe('context successors', function () { tieBreakerField: string, tieBreakerValue: number, size: number - ) => Promise; + ) => Promise; let dataPluginMock: DataPublicPluginStart; // eslint-disable-next-line @typescript-eslint/no-explicit-any let mockSearchSource: any; @@ -43,6 +44,9 @@ describe('context successors', function () { timeFieldName: '@timestamp', isTimeNanosBased: () => false, popularizeField: () => {}, + fields: { + getByName: jest.fn(), + }, } as unknown as DataView; describe('function fetchSuccessors', function () { @@ -58,17 +62,23 @@ describe('context successors', function () { } as unknown as DataPublicPluginStart; fetchSuccessors = (timeValIso, timeValNr, tieBreakerField, tieBreakerValue, size) => { - const anchor = { - _source: { - [indexPattern.timeFieldName!]: timeValIso, + const anchor = buildDataTableRecord( + { + _index: 't', + _id: '1', + _source: { + [indexPattern.timeFieldName!]: timeValIso, + }, + sort: [timeValNr, tieBreakerValue], }, - sort: [timeValNr, tieBreakerValue], - }; + indexPattern, + true + ); return fetchSurroundingDocs( SurrDocType.SUCCESSORS, indexPattern, - anchor as EsHitRecord, + anchor, tieBreakerField, SortDirection.desc, size, @@ -90,7 +100,9 @@ describe('context successors', function () { return fetchSuccessors(ANCHOR_TIMESTAMP_3000, MS_PER_DAY * 3000, '_doc', 0, 3).then( (hits) => { expect(mockSearchSource.fetch$.calledOnce).toBe(true); - expect(hits).toEqual(mockSearchSource._stubHits.slice(-3)); + expect(hits).toEqual( + buildDataTableRecordList(mockSearchSource._stubHits.slice(-3), indexPattern) + ); } ); }); @@ -120,8 +132,9 @@ describe('context successors', function () { // should have ended with a half-open interval expect(Object.keys(last(intervals) ?? {})).toEqual(['format', 'lte']); expect(intervals.length).toBeGreaterThan(1); - - expect(hits).toEqual(mockSearchSource._stubHits.slice(-3)); + expect(hits).toEqual( + buildDataTableRecordList(mockSearchSource._stubHits.slice(-3), indexPattern) + ); } ); }); @@ -149,8 +162,9 @@ describe('context successors', function () { // should have stopped before reaching MS_PER_DAY * 2200 expect(moment(last(intervals)?.gte).valueOf()).toBeGreaterThan(MS_PER_DAY * 2200); expect(intervals.length).toBeGreaterThan(1); - - expect(hits).toEqual(mockSearchSource._stubHits.slice(0, 4)); + expect(hits).toEqual( + buildDataTableRecordList(mockSearchSource._stubHits.slice(0, 4), indexPattern) + ); } ); }); @@ -194,17 +208,23 @@ describe('context successors', function () { } as unknown as DataPublicPluginStart; fetchSuccessors = (timeValIso, timeValNr, tieBreakerField, tieBreakerValue, size) => { - const anchor = { - _source: { - [indexPattern.timeFieldName!]: timeValIso, + const anchor = buildDataTableRecord( + { + _id: '1', + _index: 'test', + _source: { + [indexPattern.timeFieldName!]: timeValIso, + }, + sort: [timeValNr, tieBreakerValue], }, - sort: [timeValNr, tieBreakerValue], - }; + indexPattern, + true + ); return fetchSurroundingDocs( SurrDocType.SUCCESSORS, indexPattern, - anchor as EsHitRecord, + anchor, tieBreakerField, SortDirection.desc, size, @@ -227,7 +247,9 @@ describe('context successors', function () { return fetchSuccessors(ANCHOR_TIMESTAMP_3000, MS_PER_DAY * 3000, '_doc', 0, 3).then( (hits) => { expect(mockSearchSource.fetch$.calledOnce).toBe(true); - expect(hits).toEqual(mockSearchSource._stubHits.slice(-3)); + expect(hits).toEqual( + buildDataTableRecordList(mockSearchSource._stubHits.slice(-3), indexPattern) + ); const setFieldsSpy = mockSearchSource.setField.withArgs('fields'); const removeFieldsSpy = mockSearchSource.removeField.withArgs('fieldsFromSource'); expect(removeFieldsSpy.calledOnce).toBe(true); diff --git a/src/plugins/discover/public/application/context/services/context.ts b/src/plugins/discover/public/application/context/services/context.ts index 336ca6a21cc33..c2f2b7f765f73 100644 --- a/src/plugins/discover/public/application/context/services/context.ts +++ b/src/plugins/discover/public/application/context/services/context.ts @@ -14,7 +14,7 @@ import { fetchHitsInInterval } from '../utils/fetch_hits_in_interval'; import { generateIntervals } from '../utils/generate_intervals'; import { getEsQuerySearchAfter } from '../utils/get_es_query_search_after'; import { getEsQuerySort } from '../utils/get_es_query_sort'; -import { EsHitRecord, EsHitRecordList } from '../../types'; +import { DataTableRecord } from '../../../types'; export enum SurrDocType { SUCCESSORS = 'successors', @@ -31,7 +31,7 @@ const LOOKUP_OFFSETS = [0, 1, 7, 30, 365, 10000].map((days) => days * DAY_MILLIS * * @param {SurrDocType} type - `successors` or `predecessors` * @param {DataView} indexPattern - * @param {EsHitRecord} anchor - anchor record + * @param {DataTableRecord} anchor - anchor record * @param {string} tieBreakerField - name of the tie breaker, the 2nd sort field * @param {SortDirection} sortDir - direction of sorting * @param {number} size - number of records to retrieve @@ -42,14 +42,14 @@ const LOOKUP_OFFSETS = [0, 1, 7, 30, 365, 10000].map((days) => days * DAY_MILLIS export async function fetchSurroundingDocs( type: SurrDocType, indexPattern: DataView, - anchor: EsHitRecord, + anchor: DataTableRecord, tieBreakerField: string, sortDir: SortDirection, size: number, filters: Filter[], data: DataPublicPluginStart, useNewFieldsApi?: boolean -): Promise { +): Promise { if (typeof anchor !== 'object' || anchor === null || !size) { return []; } @@ -57,13 +57,16 @@ export async function fetchSurroundingDocs( const searchSource = data.search.searchSource.createEmpty(); updateSearchSource(searchSource, indexPattern, filters, Boolean(useNewFieldsApi)); const sortDirToApply = type === SurrDocType.SUCCESSORS ? sortDir : reverseSortDir(sortDir); + const anchorRaw = anchor.raw!; - const nanos = indexPattern.isTimeNanosBased() ? extractNanos(anchor.fields[timeField][0]) : ''; + const nanos = indexPattern.isTimeNanosBased() + ? extractNanos(anchorRaw.fields?.[timeField][0]) + : ''; const timeValueMillis = - nanos !== '' ? convertIsoToMillis(anchor.fields[timeField][0]) : anchor.sort[0]; + nanos !== '' ? convertIsoToMillis(anchorRaw.fields?.[timeField][0]) : anchorRaw.sort?.[0]; const intervals = generateIntervals(LOOKUP_OFFSETS, timeValueMillis as number, type, sortDir); - let documents: EsHitRecordList = []; + let documents: DataTableRecord[] = []; for (const interval of intervals) { const remainingSize = size - documents.length; @@ -92,7 +95,7 @@ export async function fetchSurroundingDocs( searchAfter, remainingSize, nanos, - anchor._id + anchor.raw._id ); documents = diff --git a/src/plugins/discover/public/application/context/services/context_query_state.ts b/src/plugins/discover/public/application/context/services/context_query_state.ts index 3a6a4c0959ea6..611c7666b52fb 100644 --- a/src/plugins/discover/public/application/context/services/context_query_state.ts +++ b/src/plugins/discover/public/application/context/services/context_query_state.ts @@ -6,21 +6,21 @@ * Side Public License, v 1. */ -import { EsHitRecord, EsHitRecordList } from '../../types'; +import { DataTableRecord } from '../../../types'; export interface ContextFetchState { /** * Documents listed before anchor */ - predecessors: EsHitRecordList; + predecessors: DataTableRecord[]; /** * Documents after anchor */ - successors: EsHitRecordList; + successors: DataTableRecord[]; /** * Anchor document */ - anchor: EsHitRecord; + anchor: DataTableRecord; /** * Anchor fetch status */ @@ -54,7 +54,7 @@ export interface LoadingStatusEntry { } export const getInitialContextQueryState = (): ContextFetchState => ({ - anchor: {} as EsHitRecord, + anchor: {} as DataTableRecord, predecessors: [], successors: [], anchorStatus: { value: LoadingStatus.UNINITIALIZED }, diff --git a/src/plugins/discover/public/application/context/utils/fetch_hits_in_interval.ts b/src/plugins/discover/public/application/context/utils/fetch_hits_in_interval.ts index 3547127c1ab8c..cd805a5e4f1a6 100644 --- a/src/plugins/discover/public/application/context/utils/fetch_hits_in_interval.ts +++ b/src/plugins/discover/public/application/context/utils/fetch_hits_in_interval.ts @@ -7,10 +7,11 @@ */ import { lastValueFrom } from 'rxjs'; import { ISearchSource, EsQuerySortValue, SortDirection } from '@kbn/data-plugin/public'; +import { EsQuerySearchAfter } from '@kbn/data-plugin/common'; +import { buildDataTableRecord } from '../../../utils/build_data_record'; import { convertTimeValueToIso } from './date_conversion'; import { IntervalValue } from './generate_intervals'; -import { EsQuerySearchAfter } from './get_es_query_search_after'; -import { EsHitRecord, EsHitRecordList } from '../../types'; +import type { DataTableRecord } from '../../../types'; interface RangeQuery { format: string; @@ -35,7 +36,7 @@ export async function fetchHitsInInterval( maxCount: number, nanosValue: string, anchorId: string -): Promise { +): Promise { const range: RangeQuery = { format: 'strict_date_optional_time', }; @@ -77,7 +78,8 @@ export async function fetchHitsInInterval( .fetch$(); const { rawResponse } = await lastValueFrom(fetch$); + const dataView = searchSource.getField('index'); + const records = rawResponse.hits?.hits.map((hit) => buildDataTableRecord(hit, dataView!)); - // TODO: There's a difference in the definition of SearchResponse and EsHitRecord - return (rawResponse.hits?.hits as unknown as EsHitRecord[]) || []; + return records ?? []; } diff --git a/src/plugins/discover/public/application/context/utils/get_es_query_search_after.ts b/src/plugins/discover/public/application/context/utils/get_es_query_search_after.ts index 85a68376fe43b..bfec54c61e856 100644 --- a/src/plugins/discover/public/application/context/utils/get_es_query_search_after.ts +++ b/src/plugins/discover/public/application/context/utils/get_es_query_search_after.ts @@ -5,11 +5,9 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ - +import type { EsQuerySearchAfter } from '@kbn/data-plugin/common'; import { SurrDocType } from '../services/context'; -import { EsHitRecord, EsHitRecordList } from '../../types'; - -export type EsQuerySearchAfter = [string | number, string | number]; +import type { DataTableRecord } from '../../../types'; /** * Get the searchAfter query value for elasticsearch @@ -19,9 +17,9 @@ export type EsQuerySearchAfter = [string | number, string | number]; */ export function getEsQuerySearchAfter( type: SurrDocType, - documents: EsHitRecordList, + documents: DataTableRecord[], timeFieldName: string, - anchor: EsHitRecord, + anchor: DataTableRecord, nanoSeconds: string, useNewFieldsApi?: boolean ): EsQuerySearchAfter { @@ -30,23 +28,24 @@ export function getEsQuerySearchAfter( const afterTimeRecIdx = type === SurrDocType.SUCCESSORS && documents.length ? documents.length - 1 : 0; const afterTimeDoc = documents[afterTimeRecIdx]; - let afterTimeValue = afterTimeDoc.sort[0] as string | number; + const afterTimeDocRaw = afterTimeDoc.raw; + let afterTimeValue = afterTimeDocRaw.sort?.[0] as string | number; if (nanoSeconds) { afterTimeValue = useNewFieldsApi - ? afterTimeDoc.fields[timeFieldName][0] - : afterTimeDoc._source?.[timeFieldName]; + ? afterTimeDocRaw.fields?.[timeFieldName][0] + : afterTimeDocRaw._source?.[timeFieldName]; } - return [afterTimeValue, afterTimeDoc.sort[1] as string | number]; + return [afterTimeValue, afterTimeDoc.raw.sort?.[1] as string | number]; } // if data_nanos adapt timestamp value for sorting, since numeric value was rounded by browser // ES search_after also works when number is provided as string const searchAfter = new Array(2) as EsQuerySearchAfter; - searchAfter[0] = anchor.sort[0] as string | number; + searchAfter[0] = anchor.raw.sort?.[0] as string | number; if (nanoSeconds) { searchAfter[0] = useNewFieldsApi - ? anchor.fields[timeFieldName][0] - : anchor._source?.[timeFieldName]; + ? anchor.raw.fields?.[timeFieldName][0] + : anchor.raw._source?.[timeFieldName]; } - searchAfter[1] = anchor.sort[1] as string | number; + searchAfter[1] = anchor.raw.sort?.[1] as string | number; return searchAfter; } diff --git a/src/plugins/discover/public/application/main/components/layout/discover_documents.test.tsx b/src/plugins/discover/public/application/main/components/layout/discover_documents.test.tsx index 01d5aae129a72..b76b6d6a1ce07 100644 --- a/src/plugins/discover/public/application/main/components/layout/discover_documents.test.tsx +++ b/src/plugins/discover/public/application/main/components/layout/discover_documents.test.tsx @@ -18,12 +18,13 @@ import { discoverServiceMock } from '../../../../__mocks__/services'; import { FetchStatus } from '../../../types'; import { DiscoverDocuments } from './discover_documents'; import { indexPatternMock } from '../../../../__mocks__/index_pattern'; -import { ElasticSearchHit } from '../../../../types'; import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; +import { buildDataTableRecord } from '../../../../utils/build_data_record'; +import { EsHitRecord } from '../../../../types'; setHeaderActionMenuMounter(jest.fn()); -function mountComponent(fetchStatus: FetchStatus, hits: ElasticSearchHit[]) { +function mountComponent(fetchStatus: FetchStatus, hits: EsHitRecord[]) { const services = discoverServiceMock; services.data.query.timefilter.timefilter.getTime = () => { return { from: '2020-05-14T11:05:13.590', to: '2020-05-14T11:20:13.590' }; @@ -31,7 +32,7 @@ function mountComponent(fetchStatus: FetchStatus, hits: ElasticSearchHit[]) { const documents$ = new BehaviorSubject({ fetchStatus, - result: hits, + result: hits.map((hit) => buildDataTableRecord(hit, indexPatternMock)), }) as DataDocuments$; const props = { @@ -62,13 +63,13 @@ describe('Discover documents layout', () => { }); test('render complete when loading but documents were already fetched', () => { - const component = mountComponent(FetchStatus.LOADING, esHits as ElasticSearchHit[]); + const component = mountComponent(FetchStatus.LOADING, esHits); expect(component.find('.dscDocuments__loading').exists()).toBeFalsy(); expect(component.find('.dscTable').exists()).toBeTruthy(); }); test('render complete', () => { - const component = mountComponent(FetchStatus.COMPLETE, esHits as ElasticSearchHit[]); + const component = mountComponent(FetchStatus.COMPLETE, esHits); expect(component.find('.dscDocuments__loading').exists()).toBeFalsy(); expect(component.find('.dscTable').exists()).toBeTruthy(); }); diff --git a/src/plugins/discover/public/application/main/components/layout/discover_documents.tsx b/src/plugins/discover/public/application/main/components/layout/discover_documents.tsx index 530118cb8f1c9..3dbf313fcc7c7 100644 --- a/src/plugins/discover/public/application/main/components/layout/discover_documents.tsx +++ b/src/plugins/discover/public/application/main/components/layout/discover_documents.tsx @@ -32,10 +32,10 @@ import { AppState, GetStateReturn } from '../../services/discover_state'; import { useDataState } from '../../hooks/use_data_state'; import { DocTableInfinite } from '../../../../components/doc_table/doc_table_infinite'; import { SortPairArr } from '../../../../components/doc_table/utils/get_sort'; -import { ElasticSearchHit } from '../../../../types'; import { DocumentExplorerCallout } from '../document_explorer_callout'; import { DocumentExplorerUpdateCallout } from '../document_explorer_callout/document_explorer_update_callout'; import { DiscoverTourProvider } from '../../../../components/discover_tour'; +import { DataTableRecord } from '../../../../types'; const DocTableInfiniteMemoized = React.memo(DocTableInfinite); const DataGridMemoized = React.memo(DiscoverGrid); @@ -51,12 +51,12 @@ function DiscoverDocumentsComponent({ stateContainer, }: { documents$: DataDocuments$; - expandedDoc?: ElasticSearchHit; + expandedDoc?: DataTableRecord; indexPattern: DataView; navigateTo: (url: string) => void; onAddFilter: DocViewFilterFn; savedSearch: SavedSearch; - setExpandedDoc: (doc?: ElasticSearchHit) => void; + setExpandedDoc: (doc?: DataTableRecord) => void; state: AppState; stateContainer: GetStateReturn; }) { diff --git a/src/plugins/discover/public/application/main/components/layout/discover_layout.test.tsx b/src/plugins/discover/public/application/main/components/layout/discover_layout.test.tsx index 1025270dab355..4aff6e2a78070 100644 --- a/src/plugins/discover/public/application/main/components/layout/discover_layout.test.tsx +++ b/src/plugins/discover/public/application/main/components/layout/discover_layout.test.tsx @@ -32,10 +32,10 @@ import { FetchStatus } from '../../../types'; import { RequestAdapter } from '@kbn/inspector-plugin'; import { Chart } from '../chart/point_series'; import { DiscoverSidebar } from '../sidebar/discover_sidebar'; -import { ElasticSearchHit } from '../../../../types'; import { LocalStorageMock } from '../../../../__mocks__/local_storage_mock'; import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; import { DiscoverServices } from '../../../../build_services'; +import { buildDataTableRecord } from '../../../../utils/build_data_record'; setHeaderActionMenuMounter(jest.fn()); @@ -66,7 +66,7 @@ function mountComponent( const documents$ = new BehaviorSubject({ fetchStatus: FetchStatus.COMPLETE, - result: esHits as ElasticSearchHit[], + result: esHits.map((esHit) => buildDataTableRecord(esHit, indexPattern)), }) as DataDocuments$; const availableFields$ = new BehaviorSubject({ diff --git a/src/plugins/discover/public/application/main/components/layout/types.ts b/src/plugins/discover/public/application/main/components/layout/types.ts index e7bcfb3bc3971..f381f87c7389d 100644 --- a/src/plugins/discover/public/application/main/components/layout/types.ts +++ b/src/plugins/discover/public/application/main/components/layout/types.ts @@ -11,10 +11,10 @@ import type { SavedObject } from '@kbn/data-plugin/public'; import type { DataView, DataViewAttributes } from '@kbn/data-views-plugin/public'; import { ISearchSource } from '@kbn/data-plugin/public'; import { RequestAdapter } from '@kbn/inspector-plugin'; +import { DataTableRecord } from '../../../../types'; import { AppState, GetStateReturn } from '../../services/discover_state'; import { DataRefetch$, SavedSearchData } from '../../hooks/use_saved_search'; import { SavedSearch } from '../../../../services/saved_searches'; -import { ElasticSearchHit } from '../../../../types'; export interface DiscoverLayoutProps { indexPattern: DataView; @@ -24,8 +24,8 @@ export interface DiscoverLayoutProps { onChangeIndexPattern: (id: string) => void; onUpdateQuery: (payload: { dateRange: TimeRange; query?: Query }, isUpdate?: boolean) => void; resetSavedSearch: () => void; - expandedDoc?: ElasticSearchHit; - setExpandedDoc: (doc?: ElasticSearchHit) => void; + expandedDoc?: DataTableRecord; + setExpandedDoc: (doc?: DataTableRecord) => void; savedSearch: SavedSearch; savedSearchData$: SavedSearchData; savedSearchRefetch$: DataRefetch$; diff --git a/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar.test.tsx b/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar.test.tsx index 34cfde26ff32c..2bbd2e579f4aa 100644 --- a/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar.test.tsx +++ b/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar.test.tsx @@ -5,18 +5,13 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ - -import { cloneDeep, each } from 'lodash'; import { ReactWrapper } from 'enzyme'; import { findTestSubject } from '@elastic/eui/lib/test'; import { Action } from '@kbn/ui-actions-plugin/public'; -// @ts-expect-error -import realHits from '../../../../__fixtures__/real_hits'; - +import { getDataTableRecords } from '../../../../__fixtures__/real_hits'; import { mountWithIntl } from '@kbn/test-jest-helpers'; import React from 'react'; import { DiscoverSidebarProps } from './discover_sidebar'; -import { flattenHit } from '@kbn/data-plugin/public'; import { DataViewAttributes } from '@kbn/data-views-plugin/public'; import { SavedObject } from '@kbn/core/types'; import { getDefaultFieldFilter } from './lib/field_filter'; @@ -24,7 +19,6 @@ import { DiscoverSidebarComponent as DiscoverSidebar } from './discover_sidebar' import { discoverServiceMock as mockDiscoverServices } from '../../../../__mocks__/services'; import { stubLogstashIndexPattern } from '@kbn/data-plugin/common/stubs'; import { VIEW_MODE } from '../../../../components/view_mode_toggle'; -import { ElasticSearchHit } from '../../../../types'; import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; import { BehaviorSubject } from 'rxjs'; import { FetchStatus } from '../../../types'; @@ -42,9 +36,7 @@ jest.mock('../../../../kibana_services', () => ({ function getCompProps(): DiscoverSidebarProps { const indexPattern = stubLogstashIndexPattern; - const hits = each(cloneDeep(realHits), (hit) => - flattenHit(hit, indexPattern) - ) as unknown as ElasticSearchHit[]; + const hits = getDataTableRecords(indexPattern); const indexPatternList = [ { id: '0', attributes: { title: 'b' } } as SavedObject, @@ -55,7 +47,7 @@ function getCompProps(): DiscoverSidebarProps { const fieldCounts: Record = {}; for (const hit of hits) { - for (const key of Object.keys(flattenHit(hit, indexPattern))) { + for (const key of Object.keys(hit.flattened)) { fieldCounts[key] = (fieldCounts[key] || 0) + 1; } } diff --git a/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar.tsx b/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar.tsx index 0141aead76eff..5ce3ad6cd147d 100644 --- a/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar.tsx +++ b/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar.tsx @@ -40,7 +40,7 @@ import { getIndexPatternFieldList } from './lib/get_index_pattern_field_list'; import { DiscoverSidebarResponsiveProps } from './discover_sidebar_responsive'; import { VIEW_MODE } from '../../../../components/view_mode_toggle'; import { DISCOVER_TOUR_STEP_ANCHOR_IDS } from '../../../../components/discover_tour'; -import { ElasticSearchHit } from '../../../../types'; +import type { DataTableRecord } from '../../../../types'; /** * Default number of available fields displayed and added on scroll @@ -88,7 +88,7 @@ export interface DiscoverSidebarProps extends Omit ({ function getCompProps(): DiscoverSidebarResponsiveProps { const indexPattern = stubLogstashIndexPattern; - const hits = each(cloneDeep(realHits), (hit) => - flattenHit(hit, indexPattern) - ) as unknown as ElasticSearchHit[]; + const hits = getDataTableRecords(indexPattern); const indexPatternList = [ { id: '0', attributes: { title: 'b' } } as SavedObject, @@ -90,7 +84,7 @@ function getCompProps(): DiscoverSidebarResponsiveProps { ]; for (const hit of hits) { - for (const key of Object.keys(flattenHit(hit, indexPattern))) { + for (const key of Object.keys(hit.flattened)) { mockfieldCounts[key] = (mockfieldCounts[key] || 0) + 1; } } @@ -99,7 +93,7 @@ function getCompProps(): DiscoverSidebarResponsiveProps { columns: ['extension'], documents$: new BehaviorSubject({ fetchStatus: FetchStatus.COMPLETE, - result: hits as ElasticSearchHit[], + result: hits, }) as DataDocuments$, availableFields$: new BehaviorSubject({ fetchStatus: FetchStatus.COMPLETE, diff --git a/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar_responsive.tsx b/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar_responsive.tsx index c6d8d05a23ad8..e4134184306fa 100644 --- a/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar_responsive.tsx +++ b/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar_responsive.tsx @@ -123,7 +123,10 @@ export function DiscoverSidebarResponsive(props: DiscoverSidebarResponsiveProps) */ const fieldCounts = useRef | null>(null); if (fieldCounts.current === null) { - fieldCounts.current = calcFieldCounts(props.documents$.getValue().result, selectedIndexPattern); + fieldCounts.current = calcFieldCounts( + props.documents$.getValue().result!, + selectedIndexPattern + ); } const [documentState, setDocumentState] = useState(props.documents$.getValue()); @@ -266,7 +269,7 @@ export function DiscoverSidebarResponsive(props: DiscoverSidebarResponsiveProps) flattenHit(hit, dataView)); + hits = getDataTableRecords(dataView); }); it('Should return an array of values for _source fields', function () { @@ -153,7 +149,7 @@ describe('fieldCalculator', function () { let params: { hits: any; field: any; count: number; dataView: DataView }; beforeEach(function () { params = { - hits: cloneDeep(realHits), + hits: getDataTableRecords(dataView), field: dataView.fields.getByName('extension'), count: 3, dataView, diff --git a/src/plugins/discover/public/application/main/components/sidebar/lib/get_details.ts b/src/plugins/discover/public/application/main/components/sidebar/lib/get_details.ts index 78e752494c43d..99ab866a53491 100644 --- a/src/plugins/discover/public/application/main/components/sidebar/lib/get_details.ts +++ b/src/plugins/discover/public/application/main/components/sidebar/lib/get_details.ts @@ -9,11 +9,11 @@ import { DataView, DataViewField } from '@kbn/data-views-plugin/public'; // @ts-expect-error import { fieldCalculator } from './field_calculator'; -import { ElasticSearchHit } from '../../../../../types'; +import { DataTableRecord } from '../../../../../types'; export function getDetails( field: DataViewField, - hits: ElasticSearchHit[] | undefined, + hits: DataTableRecord[] | undefined, columns: string[], indexPattern?: DataView ) { @@ -24,7 +24,6 @@ export function getDetails( ...fieldCalculator.getFieldValueCounts({ hits, field, - indexPattern, count: 5, grouped: false, }), diff --git a/src/plugins/discover/public/application/main/discover_main_app.tsx b/src/plugins/discover/public/application/main/discover_main_app.tsx index ae477719af119..6025c27a0c433 100644 --- a/src/plugins/discover/public/application/main/discover_main_app.tsx +++ b/src/plugins/discover/public/application/main/discover_main_app.tsx @@ -15,8 +15,8 @@ import { addHelpMenuToAppChrome } from '../../components/help_menu/help_menu_uti import { useDiscoverState } from './hooks/use_discover_state'; import { useUrl } from './hooks/use_url'; import { SavedSearch } from '../../services/saved_searches'; -import { ElasticSearchHit } from '../../types'; import { useDiscoverServices } from '../../hooks/use_discover_services'; +import { DataTableRecord } from '../../types'; const DiscoverLayoutMemoized = React.memo(DiscoverLayout); @@ -36,7 +36,7 @@ export function DiscoverMainApp(props: DiscoverMainProps) { const services = useDiscoverServices(); const { chrome, docLinks, uiSettings: config, data } = services; const history = useHistory(); - const [expandedDoc, setExpandedDoc] = useState(undefined); + const [expandedDoc, setExpandedDoc] = useState(undefined); const navigateTo = useCallback( (path: string) => { history.push(path); diff --git a/src/plugins/discover/public/application/main/hooks/use_discover_state.ts b/src/plugins/discover/public/application/main/hooks/use_discover_state.ts index ab481f2f67a50..2d82e12824f04 100644 --- a/src/plugins/discover/public/application/main/hooks/use_discover_state.ts +++ b/src/plugins/discover/public/application/main/hooks/use_discover_state.ts @@ -24,7 +24,7 @@ import { useSearchSession } from './use_search_session'; import { FetchStatus } from '../../types'; import { getSwitchIndexPatternAppState } from '../utils/get_switch_index_pattern_app_state'; import { SortPairArr } from '../../../components/doc_table/utils/get_sort'; -import { ElasticSearchHit } from '../../../types'; +import { DataTableRecord } from '../../../types'; export function useDiscoverState({ services, @@ -35,7 +35,7 @@ export function useDiscoverState({ services: DiscoverServices; savedSearch: SavedSearch; history: History; - setExpandedDoc: (doc?: ElasticSearchHit) => void; + setExpandedDoc: (doc?: DataTableRecord) => void; }) { const { uiSettings: config, data, filterManager, indexPatterns, storage } = services; const useNewFieldsApi = useMemo(() => !config.get(SEARCH_FIELDS_FROM_SOURCE), [config]); diff --git a/src/plugins/discover/public/application/main/hooks/use_saved_search.ts b/src/plugins/discover/public/application/main/hooks/use_saved_search.ts index d4d6b869c7ee7..0cfa1b2e97579 100644 --- a/src/plugins/discover/public/application/main/hooks/use_saved_search.ts +++ b/src/plugins/discover/public/application/main/hooks/use_saved_search.ts @@ -7,9 +7,9 @@ */ import { useCallback, useEffect, useMemo, useRef } from 'react'; import { BehaviorSubject, Subject } from 'rxjs'; +import type { AutoRefreshDoneFn } from '@kbn/data-plugin/public'; import { ISearchSource } from '@kbn/data-plugin/public'; import { RequestAdapter } from '@kbn/inspector-plugin/public'; -import type { AutoRefreshDoneFn } from '@kbn/data-plugin/public'; import { DiscoverServices } from '../../../build_services'; import { DiscoverSearchSessionManager } from '../services/discover_search_session'; import { GetStateReturn } from '../services/discover_state'; @@ -17,13 +17,12 @@ import { validateTimeRange } from '../utils/validate_time_range'; import { Chart } from '../components/chart/point_series'; import { useSingleton } from './use_singleton'; import { FetchStatus } from '../../types'; - import { fetchAll } from '../utils/fetch_all'; import { useBehaviorSubject } from './use_behavior_subject'; import { sendResetMsg } from './use_saved_search_messages'; import { getFetch$ } from '../utils/get_fetch_observable'; -import { ElasticSearchHit } from '../../../types'; import { SavedSearch } from '../../../services/saved_searches'; +import type { DataTableRecord } from '../../../types'; export interface SavedSearchData { main$: DataMain$; @@ -66,7 +65,7 @@ export interface DataMainMsg extends DataMsg { } export interface DataDocumentsMsg extends DataMsg { - result?: ElasticSearchHit[]; + result?: DataTableRecord[]; } export interface DataTotalHitsMsg extends DataMsg { diff --git a/src/plugins/discover/public/application/main/utils/calc_field_counts.test.ts b/src/plugins/discover/public/application/main/utils/calc_field_counts.test.ts index 2ed564194bd25..1534dbf6aad4d 100644 --- a/src/plugins/discover/public/application/main/utils/calc_field_counts.test.ts +++ b/src/plugins/discover/public/application/main/utils/calc_field_counts.test.ts @@ -8,19 +8,17 @@ import { calcFieldCounts } from './calc_field_counts'; import { indexPatternMock } from '../../../__mocks__/index_pattern'; -import { ElasticSearchHit } from '../../../types'; +import { buildDataTableRecord } from '../../../utils/build_data_record'; describe('calcFieldCounts', () => { test('returns valid field count data', async () => { const rows = [ - { _id: 1, _source: { message: 'test1', bytes: 20 } }, - { _id: 2, _source: { name: 'test2', extension: 'jpg' } }, - ] as unknown as ElasticSearchHit[]; + { _id: '1', _index: 'test', _source: { message: 'test1', bytes: 20 } }, + { _id: '2', _index: 'test', _source: { name: 'test2', extension: 'jpg' } }, + ].map((row) => buildDataTableRecord(row)); const result = calcFieldCounts(rows, indexPatternMock); expect(result).toMatchInlineSnapshot(` Object { - "_index": 2, - "_score": 2, "bytes": 1, "extension": 1, "message": 1, @@ -30,14 +28,12 @@ describe('calcFieldCounts', () => { }); test('updates field count data', async () => { const rows = [ - { _id: 1, _source: { message: 'test1', bytes: 20 } }, - { _id: 2, _source: { name: 'test2', extension: 'jpg' } }, - ] as unknown as ElasticSearchHit[]; + { _id: '1', _index: 'test', _source: { message: 'test1', bytes: 20 } }, + { _id: '2', _index: 'test', _source: { name: 'test2', extension: 'jpg' } }, + ].map((row) => buildDataTableRecord(row)); const result = calcFieldCounts(rows, indexPatternMock); expect(result).toMatchInlineSnapshot(` Object { - "_index": 2, - "_score": 2, "bytes": 1, "extension": 1, "message": 1, diff --git a/src/plugins/discover/public/application/main/utils/calc_field_counts.ts b/src/plugins/discover/public/application/main/utils/calc_field_counts.ts index cf2b5d7a880b3..5112625ba12b9 100644 --- a/src/plugins/discover/public/application/main/utils/calc_field_counts.ts +++ b/src/plugins/discover/public/application/main/utils/calc_field_counts.ts @@ -6,20 +6,19 @@ * Side Public License, v 1. */ import { DataView } from '@kbn/data-views-plugin/public'; -import { flattenHit } from '@kbn/data-plugin/public'; -import { ElasticSearchHit } from '../../../types'; +import { DataTableRecord } from '../../../types'; /** * This function is calculating stats of the available fields, for usage in sidebar and sharing * Note that this values aren't displayed, but used for internal calculations */ -export function calcFieldCounts(rows?: ElasticSearchHit[], indexPattern?: DataView) { +export function calcFieldCounts(rows?: DataTableRecord[], indexPattern?: DataView) { const counts: Record = {}; if (!rows || !indexPattern) { return {}; } for (const hit of rows) { - const fields = Object.keys(flattenHit(hit, indexPattern, { includeIgnoredValues: true })); + const fields = Object.keys(hit.flattened); for (const fieldName of fields) { counts[fieldName] = (counts[fieldName] || 0) + 1; } diff --git a/src/plugins/discover/public/application/main/utils/fetch_all.test.ts b/src/plugins/discover/public/application/main/utils/fetch_all.test.ts index d372f921f9976..e33d931c571da 100644 --- a/src/plugins/discover/public/application/main/utils/fetch_all.test.ts +++ b/src/plugins/discover/public/application/main/utils/fetch_all.test.ts @@ -27,6 +27,8 @@ import { import { fetchDocuments } from './fetch_documents'; import { fetchChart } from './fetch_chart'; import { fetchTotalHits } from './fetch_total_hits'; +import { indexPatternMock } from '../../../__mocks__/index_pattern'; +import { buildDataTableRecord } from '../../../utils/build_data_record'; jest.mock('./fetch_documents', () => ({ fetchDocuments: jest.fn().mockResolvedValue([]), @@ -119,7 +121,10 @@ describe('test fetchAll', () => { expect(await collect()).toEqual([ { fetchStatus: FetchStatus.UNINITIALIZED }, { fetchStatus: FetchStatus.LOADING }, - { fetchStatus: FetchStatus.COMPLETE, result: hits }, + { + fetchStatus: FetchStatus.COMPLETE, + result: hits.map((hit) => buildDataTableRecord(hit, indexPatternMock)), + }, ]); }); diff --git a/src/plugins/discover/public/application/main/utils/fetch_all.ts b/src/plugins/discover/public/application/main/utils/fetch_all.ts index 1d5d4646445a5..655027dddbf1e 100644 --- a/src/plugins/discover/public/application/main/utils/fetch_all.ts +++ b/src/plugins/discover/public/application/main/utils/fetch_all.ts @@ -5,11 +5,11 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -import { ISearchSource } from '@kbn/data-plugin/public'; +import { DataPublicPluginStart, ISearchSource } from '@kbn/data-plugin/public'; import { Adapters } from '@kbn/inspector-plugin'; -import { DataPublicPluginStart } from '@kbn/data-plugin/public'; import { ReduxLikeStateContainer } from '@kbn/kibana-utils-plugin/common'; import { DataViewType } from '@kbn/data-views-plugin/public'; +import { buildDataTableRecord } from '../../../utils/build_data_record'; import { sendCompleteMsg, sendErrorMsg, @@ -45,6 +45,7 @@ export interface FetchDeps { services: DiscoverServices; useNewFieldsApi: boolean; } + /** * This function starts fetching all required queries in Discover. This will be the query to load the individual * documents, and depending on whether a chart is shown either the aggregation query to load the chart data @@ -139,10 +140,13 @@ export function fetchAll( result: docs.length, }); } + const dataView = searchSource.getField('index')!; + + const resultDocs = docs.map((doc) => buildDataTableRecord(doc, dataView)); dataSubjects.documents$.next({ fetchStatus: FetchStatus.COMPLETE, - result: docs, + result: resultDocs, }); checkHitCount(docs.length); diff --git a/src/plugins/discover/public/application/main/utils/fetch_chart.ts b/src/plugins/discover/public/application/main/utils/fetch_chart.ts index 5117bffe5c9b8..da1a071a18d4b 100644 --- a/src/plugins/discover/public/application/main/utils/fetch_chart.ts +++ b/src/plugins/discover/public/application/main/utils/fetch_chart.ts @@ -28,14 +28,7 @@ interface Result { export function fetchChart( searchSource: ISearchSource, - { - abortController, - appStateContainer, - data, - inspectorAdapters, - searchSessionId, - savedSearch, - }: FetchDeps + { abortController, appStateContainer, data, inspectorAdapters, searchSessionId }: FetchDeps ): Promise { const interval = appStateContainer.getState().interval ?? 'auto'; const chartAggConfigs = updateSearchSource(searchSource, interval, data); diff --git a/src/plugins/discover/public/application/main/utils/fetch_documents.ts b/src/plugins/discover/public/application/main/utils/fetch_documents.ts index d652f544a8f70..e09875d11deb6 100644 --- a/src/plugins/discover/public/application/main/utils/fetch_documents.ts +++ b/src/plugins/discover/public/application/main/utils/fetch_documents.ts @@ -18,7 +18,7 @@ import { FetchDeps } from './fetch_all'; */ export const fetchDocuments = ( searchSource: ISearchSource, - { abortController, inspectorAdapters, searchSessionId, services, savedSearch }: FetchDeps + { abortController, inspectorAdapters, searchSessionId, services }: FetchDeps ) => { searchSource.setField('size', services.uiSettings.get(SAMPLE_SIZE_SETTING)); searchSource.setField('trackTotalHits', false); diff --git a/src/plugins/discover/public/application/types.ts b/src/plugins/discover/public/application/types.ts index f04f3bf77c2f9..798e0f350cc5f 100644 --- a/src/plugins/discover/public/application/types.ts +++ b/src/plugins/discover/public/application/types.ts @@ -5,7 +5,6 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; export enum FetchStatus { UNINITIALIZED = 'uninitialized', @@ -14,13 +13,3 @@ export enum FetchStatus { COMPLETE = 'complete', ERROR = 'error', } - -export type EsHitRecord = Required< - Pick -> & { - _source?: Record; - _score?: number; - // note that this a special property for Discover Context, to determine the anchor record - isAnchor?: boolean; -}; -export type EsHitRecordList = EsHitRecord[]; diff --git a/src/plugins/discover/public/components/discover_grid/discover_grid.test.tsx b/src/plugins/discover/public/components/discover_grid/discover_grid.test.tsx index c5ac7335b69c2..876ffadfed433 100644 --- a/src/plugins/discover/public/components/discover_grid/discover_grid.test.tsx +++ b/src/plugins/discover/public/components/discover_grid/discover_grid.test.tsx @@ -14,10 +14,11 @@ import { esHits } from '../../__mocks__/es_hits'; import { indexPatternMock } from '../../__mocks__/index_pattern'; import { mountWithIntl } from '@kbn/test-jest-helpers'; import { DiscoverGrid, DiscoverGridProps } from './discover_grid'; -import { getDocId } from './discover_grid_document_selection'; -import { ElasticSearchHit } from '../../types'; import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; import { discoverServiceMock } from '../../__mocks__/services'; +import { buildDataTableRecord } from '../../utils/build_data_record'; +import { getDocId } from '../../utils/get_doc_id'; +import { EsHitRecord } from '../../types'; function getProps() { return { @@ -32,7 +33,7 @@ function getProps() { onResize: jest.fn(), onSetColumns: jest.fn(), onSort: jest.fn(), - rows: esHits, + rows: esHits.map((hit) => buildDataTableRecord(hit, indexPatternMock)), sampleSize: 30, searchDescription: '', searchTitle: '', @@ -74,7 +75,7 @@ function getDisplayedDocNr(component: ReactWrapper) { async function toggleDocSelection( component: ReactWrapper, - document: ElasticSearchHit + document: EsHitRecord ) { act(() => { const docId = getDocId(document); @@ -146,7 +147,7 @@ describe('DiscoverGrid', () => { bytes: 50, }, }, - ], + ].map((row) => buildDataTableRecord(row, indexPatternMock)), }); expect(getDisplayedDocNr(component)).toBe(1); expect(getSelectedDocNr(component)).toBe(0); diff --git a/src/plugins/discover/public/components/discover_grid/discover_grid.tsx b/src/plugins/discover/public/components/discover_grid/discover_grid.tsx index 48fd2ba656746..5b60a92110320 100644 --- a/src/plugins/discover/public/components/discover_grid/discover_grid.tsx +++ b/src/plugins/discover/public/components/discover_grid/discover_grid.tsx @@ -23,7 +23,6 @@ import { EuiLink, } from '@elastic/eui'; import type { DataView } from '@kbn/data-views-plugin/public'; -import { flattenHit } from '@kbn/data-plugin/public'; import { DocViewFilterFn } from '../../services/doc_views/doc_views_types'; import { getSchemaDetectors } from './discover_grid_schema'; import { DiscoverGridFlyout } from './discover_grid_flyout'; @@ -48,10 +47,10 @@ import { MAX_DOC_FIELDS_DISPLAYED, SHOW_MULTIFIELDS, } from '../../../common'; -import { DiscoverGridDocumentToolbarBtn, getDocId } from './discover_grid_document_selection'; +import { DiscoverGridDocumentToolbarBtn } from './discover_grid_document_selection'; import { SortPairArr } from '../doc_table/utils/get_sort'; import { getFieldsToShow } from '../../utils/get_fields_to_show'; -import type { ElasticSearchHit, ValueToStringConverter } from '../../types'; +import type { DataTableRecord, ValueToStringConverter } from '../../types'; import { useRowHeightsOptions } from '../../hooks/use_row_heights_options'; import { useDiscoverServices } from '../../hooks/use_discover_services'; import { convertValueToString } from '../../utils/convert_value_to_string'; @@ -77,7 +76,7 @@ export interface DiscoverGridProps { /** * If set, the given document is displayed in a flyout */ - expandedDoc?: ElasticSearchHit; + expandedDoc?: DataTableRecord; /** * The used index pattern */ @@ -114,7 +113,7 @@ export interface DiscoverGridProps { /** * Array of documents provided by Elasticsearch */ - rows?: ElasticSearchHit[]; + rows?: DataTableRecord[]; /** * The max size of the documents returned by Elasticsearch */ @@ -122,7 +121,7 @@ export interface DiscoverGridProps { /** * Function to set the expanded document, which is displayed in a flyout */ - setExpandedDoc: (doc?: ElasticSearchHit) => void; + setExpandedDoc: (doc?: DataTableRecord) => void; /** * Grid display settings persisted in Elasticsearch (e.g. column width) */ @@ -211,7 +210,7 @@ export const DiscoverGrid = ({ if (!selectedDocs.length || !rows?.length) { return []; } - const idMap = rows.reduce((map, row) => map.set(getDocId(row), true), new Map()); + const idMap = rows.reduce((map, row) => map.set(row.id, true), new Map()); // filter out selected docs that are no longer part of the current data const result = selectedDocs.filter((docId) => idMap.get(docId)); if (result.length === 0 && isFilterActive) { @@ -227,7 +226,7 @@ export const DiscoverGrid = ({ if (!isFilterActive || usedSelectedDocs.length === 0) { return rows; } - const rowsFiltered = rows.filter((row) => usedSelectedDocs.includes(getDocId(row))); + const rowsFiltered = rows.filter((row) => usedSelectedDocs.includes(row.id)); if (!rowsFiltered.length) { // in case the selected docs are no longer part of the sample of 500, show all docs return rows; @@ -235,25 +234,18 @@ export const DiscoverGrid = ({ return rowsFiltered; }, [rows, usedSelectedDocs, isFilterActive]); - const displayedRowsFlattened = useMemo(() => { - return displayedRows.map((hit) => { - return flattenHit(hit, indexPattern, { includeIgnoredValues: true }); - }); - }, [displayedRows, indexPattern]); - const valueToStringConverter: ValueToStringConverter = useCallback( (rowIndex, columnId, options) => { return convertValueToString({ rowIndex, rows: displayedRows, - rowsFlattened: displayedRowsFlattened, dataView: indexPattern, columnId, services, options, }); }, - [displayedRows, displayedRowsFlattened, indexPattern, services] + [displayedRows, indexPattern, services] ); /** @@ -314,20 +306,12 @@ export const DiscoverGrid = ({ getRenderCellValueFn( indexPattern, displayedRows, - displayedRowsFlattened, useNewFieldsApi, fieldsToShow, services.uiSettings.get(MAX_DOC_FIELDS_DISPLAYED), () => dataGridRef.current?.closeCellPopover() ), - [ - indexPattern, - displayedRowsFlattened, - displayedRows, - useNewFieldsApi, - fieldsToShow, - services.uiSettings, - ] + [indexPattern, displayedRows, useNewFieldsApi, fieldsToShow, services.uiSettings] ); /** @@ -473,7 +457,6 @@ export const DiscoverGrid = ({ expanded: expandedDoc, setExpanded: setExpandedDoc, rows: displayedRows, - rowsFlattened: displayedRowsFlattened, onFilter, indexPattern, isDarkMode: services.uiSettings.get('theme:darkMode'), diff --git a/src/plugins/discover/public/components/discover_grid/discover_grid_cell_actions.tsx b/src/plugins/discover/public/components/discover_grid/discover_grid_cell_actions.tsx index f7497dd5d459d..055967d6440a2 100644 --- a/src/plugins/discover/public/components/discover_grid/discover_grid_cell_actions.tsx +++ b/src/plugins/discover/public/components/discover_grid/discover_grid_cell_actions.tsx @@ -20,8 +20,8 @@ function onFilterCell( columnId: EuiDataGridColumnCellActionProps['columnId'], mode: '+' | '-' ) { - const row = context.rowsFlattened[rowIndex]; - const value = String(row[columnId]); + const row = context.rows[rowIndex]; + const value = String(row.flattened[columnId]); const field = context.indexPattern.fields.getByName(columnId); if (value && field) { diff --git a/src/plugins/discover/public/components/discover_grid/discover_grid_context.tsx b/src/plugins/discover/public/components/discover_grid/discover_grid_context.tsx index ca2a0ee5839cb..0761e4c40376e 100644 --- a/src/plugins/discover/public/components/discover_grid/discover_grid_context.tsx +++ b/src/plugins/discover/public/components/discover_grid/discover_grid_context.tsx @@ -9,13 +9,12 @@ import React from 'react'; import type { DataView } from '@kbn/data-views-plugin/public'; import type { DocViewFilterFn } from '../../services/doc_views/doc_views_types'; -import type { ElasticSearchHit, HitsFlattened, ValueToStringConverter } from '../../types'; +import type { DataTableRecord, ValueToStringConverter } from '../../types'; export interface GridContext { - expanded?: ElasticSearchHit; - setExpanded: (hit?: ElasticSearchHit) => void; - rows: ElasticSearchHit[]; - rowsFlattened: HitsFlattened; + expanded?: DataTableRecord | undefined; + setExpanded: (hit?: DataTableRecord) => void; + rows: DataTableRecord[]; onFilter: DocViewFilterFn; indexPattern: DataView; isDarkMode: boolean; diff --git a/src/plugins/discover/public/components/discover_grid/discover_grid_document_selection.test.tsx b/src/plugins/discover/public/components/discover_grid/discover_grid_document_selection.test.tsx index 5ca7b84789e91..96bc9082d9e0f 100644 --- a/src/plugins/discover/public/components/discover_grid/discover_grid_document_selection.test.tsx +++ b/src/plugins/discover/public/components/discover_grid/discover_grid_document_selection.test.tsx @@ -8,13 +8,10 @@ import React from 'react'; import { mountWithIntl } from '@kbn/test-jest-helpers'; import { findTestSubject } from '@elastic/eui/lib/test'; -import { - DiscoverGridDocumentToolbarBtn, - getDocId, - SelectButton, -} from './discover_grid_document_selection'; +import { DiscoverGridDocumentToolbarBtn, SelectButton } from './discover_grid_document_selection'; import { discoverGridContextMock } from '../../__mocks__/grid_context'; import { DiscoverGridContext } from './discover_grid_context'; +import { getDocId } from '../../utils/get_doc_id'; describe('document selection', () => { describe('getDocId', () => { diff --git a/src/plugins/discover/public/components/discover_grid/discover_grid_document_selection.tsx b/src/plugins/discover/public/components/discover_grid/discover_grid_document_selection.tsx index efe8784028cbf..c17ff66ac0806 100644 --- a/src/plugins/discover/public/components/discover_grid/discover_grid_document_selection.tsx +++ b/src/plugins/discover/public/components/discover_grid/discover_grid_document_selection.tsx @@ -5,37 +5,28 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -import React, { useCallback, useState, useContext, useMemo, useEffect } from 'react'; +import React, { useCallback, useContext, useEffect, useMemo, useState } from 'react'; import classNames from 'classnames'; import { EuiButtonEmpty, + EuiCheckbox, EuiContextMenuItem, EuiContextMenuPanel, EuiCopy, - EuiPopover, - EuiCheckbox, EuiDataGridCellValueElementProps, + EuiPopover, } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n-react'; -import { euiLightVars as themeLight, euiDarkVars as themeDark } from '@kbn/ui-theme'; +import { euiDarkVars as themeDark, euiLightVars as themeLight } from '@kbn/ui-theme'; import { i18n } from '@kbn/i18n'; import { DiscoverGridContext } from './discover_grid_context'; -import { ElasticSearchHit } from '../../types'; +import type { DataTableRecord } from '../../types'; -/** - * Returning a generated id of a given ES document, since `_id` can be the same - * when using different indices and shard routing - */ -export const getDocId = (doc: ElasticSearchHit & { _routing?: string }) => { - const routing = doc._routing ? doc._routing : ''; - return [doc._index, doc._id, routing].join('::'); -}; export const SelectButton = ({ rowIndex, setCellProps }: EuiDataGridCellValueElementProps) => { const { selectedDocs, expanded, rows, isDarkMode, setSelectedDocs } = useContext(DiscoverGridContext); const doc = useMemo(() => rows[rowIndex], [rows, rowIndex]); - const id = useMemo(() => getDocId(doc), [doc]); - const checked = useMemo(() => selectedDocs.includes(id), [selectedDocs, id]); + const checked = useMemo(() => selectedDocs.includes(doc.id), [selectedDocs, doc.id]); const toggleDocumentSelectionLabel = i18n.translate('discover.grid.selectDoc', { defaultMessage: `Select document '{rowNumber}'`, @@ -43,7 +34,7 @@ export const SelectButton = ({ rowIndex, setCellProps }: EuiDataGridCellValueEle }); useEffect(() => { - if (expanded && doc && expanded._id === doc._id && expanded._index === doc._index) { + if (expanded && doc && expanded.id === doc.id) { setCellProps({ style: { backgroundColor: isDarkMode ? themeDark.euiColorHighlight : themeLight.euiColorHighlight, @@ -56,16 +47,16 @@ export const SelectButton = ({ rowIndex, setCellProps }: EuiDataGridCellValueEle return ( { if (checked) { - const newSelection = selectedDocs.filter((docId) => docId !== id); + const newSelection = selectedDocs.filter((docId) => docId !== doc.id); setSelectedDocs(newSelection); } else { - setSelectedDocs([...selectedDocs, id]); + setSelectedDocs([...selectedDocs, doc.id]); } }} /> @@ -80,7 +71,7 @@ export function DiscoverGridDocumentToolbarBtn({ setSelectedDocs, }: { isFilterActive: boolean; - rows: ElasticSearchHit[]; + rows: DataTableRecord[]; selectedDocs: string[]; setIsFilterActive: (value: boolean) => void; setSelectedDocs: (value: string[]) => void; @@ -121,7 +112,11 @@ export function DiscoverGridDocumentToolbarBtn({ key="copyJsonWrapper" data-test-subj="dscGridCopySelectedDocumentsJSON" textToCopy={ - rows ? JSON.stringify(rows.filter((row) => selectedDocs.includes(getDocId(row)))) : '' + rows + ? JSON.stringify( + rows.filter((row) => selectedDocs.includes(row.id)).map((row) => row.raw) + ) + : '' } > {(copy) => ( diff --git a/src/plugins/discover/public/components/discover_grid/discover_grid_expand_button.tsx b/src/plugins/discover/public/components/discover_grid/discover_grid_expand_button.tsx index 3a506c063c177..e00e722f9c2e9 100644 --- a/src/plugins/discover/public/components/discover_grid/discover_grid_expand_button.tsx +++ b/src/plugins/discover/public/components/discover_grid/discover_grid_expand_button.tsx @@ -11,7 +11,6 @@ import { EuiButtonIcon, EuiDataGridCellValueElementProps, EuiToolTip } from '@el import { euiLightVars as themeLight, euiDarkVars as themeDark } from '@kbn/ui-theme'; import { i18n } from '@kbn/i18n'; import { DiscoverGridContext } from './discover_grid_context'; -import { EsHitRecord } from '../../application/types'; import { DISCOVER_TOUR_STEP_ANCHOR_IDS } from '../discover_tour'; /** @@ -21,16 +20,11 @@ export const ExpandButton = ({ rowIndex, setCellProps }: EuiDataGridCellValueEle const { expanded, setExpanded, rows, isDarkMode } = useContext(DiscoverGridContext); const current = rows[rowIndex]; useEffect(() => { - if ((current as EsHitRecord).isAnchor) { + if (current.isAnchor) { setCellProps({ className: 'dscDocsGrid__cell--highlight', }); - } else if ( - expanded && - current && - expanded._id === current._id && - expanded._index === current._index - ) { + } else if (expanded && current && expanded.id === current.id) { setCellProps({ style: { backgroundColor: isDarkMode ? themeDark.euiColorHighlight : themeLight.euiColorHighlight, @@ -46,7 +40,7 @@ export const ExpandButton = ({ rowIndex, setCellProps }: EuiDataGridCellValueEle defaultMessage: 'Toggle dialog with details', }); - const testSubj = (current as EsHitRecord).isAnchor + const testSubj = current.isAnchor ? 'docTableExpandToggleColumnAnchor' : 'docTableExpandToggleColumn'; diff --git a/src/plugins/discover/public/components/discover_grid/discover_grid_flyout.test.tsx b/src/plugins/discover/public/components/discover_grid/discover_grid_flyout.test.tsx index 6452349bbdc77..5b4d4d3109e19 100644 --- a/src/plugins/discover/public/components/discover_grid/discover_grid_flyout.test.tsx +++ b/src/plugins/discover/public/components/discover_grid/discover_grid_flyout.test.tsx @@ -19,7 +19,8 @@ import { setDocViewsRegistry } from '../../kibana_services'; import { indexPatternWithTimefieldMock } from '../../__mocks__/index_pattern_with_timefield'; import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; import type { DataView } from '@kbn/data-views-plugin/public'; -import type { ElasticSearchHit } from '../../types'; +import type { DataTableRecord, EsHitRecord } from '../../types'; +import { buildDataTableRecord } from '../../utils/build_data_record'; describe('Discover flyout', function () { setDocViewsRegistry(new DocViewsRegistry()); @@ -30,7 +31,7 @@ describe('Discover flyout', function () { hitIndex, }: { indexPattern?: DataView; - hits?: ElasticSearchHit[]; + hits?: DataTableRecord[]; hitIndex?: number; }) => { const onClose = jest.fn(); @@ -40,11 +41,20 @@ describe('Discover flyout', function () { history: () => ({ location: {} }), } as unknown as DiscoverServices; + const hit = buildDataTableRecord( + hitIndex ? esHits[hitIndex] : (esHits[0] as EsHitRecord), + indexPatternMock + ); + const props = { columns: ['date'], indexPattern: indexPattern || indexPatternMock, - hit: hitIndex ? esHits[hitIndex] : esHits[0], - hits: hits || esHits, + hit, + hits: + hits || + esHits.map((entry: EsHitRecord) => + buildDataTableRecord(entry, indexPattern || indexPatternMock) + ), onAddColumn: jest.fn(), onClose, onFilter: jest.fn(), @@ -116,7 +126,7 @@ describe('Discover flyout', function () { _type: '_doc', _source: { date: '2020-20-01T12:12:12.124', name: 'test2', extension: 'jpg' }, }, - ]; + ].map((hit) => buildDataTableRecord(hit, indexPatternMock)); const { component } = mountComponent({ hits }); const docNav = findTestSubject(component, 'dscDocNavigation'); expect(docNav.length).toBeFalsy(); @@ -127,7 +137,7 @@ describe('Discover flyout', function () { const { component, props } = mountComponent({}); findTestSubject(component, 'pagination-button-next').simulate('click'); // we selected 1, so we'd expect 2 - expect(props.setExpandedDoc.mock.calls[0][0]._id).toBe('2'); + expect(props.setExpandedDoc.mock.calls[0][0].raw._id).toBe('2'); }); it('doesnt allow you to navigate to the previous doc, if expanded doc is the first', async () => { @@ -149,16 +159,16 @@ describe('Discover flyout', function () { const { component, props } = mountComponent({ hitIndex: esHits.length - 1 }); findTestSubject(component, 'pagination-button-previous').simulate('click'); expect(props.setExpandedDoc).toHaveBeenCalledTimes(1); - expect(props.setExpandedDoc.mock.calls[0][0]._id).toBe('4'); + expect(props.setExpandedDoc.mock.calls[0][0].raw._id).toBe('4'); }); it('allows navigating with arrow keys through documents', () => { const { component, props } = mountComponent({}); findTestSubject(component, 'docTableDetailsFlyout').simulate('keydown', { key: 'ArrowRight' }); - expect(props.setExpandedDoc).toHaveBeenCalledWith(expect.objectContaining({ _id: '2' })); + expect(props.setExpandedDoc).toHaveBeenCalledWith(expect.objectContaining({ id: 'i::2::' })); component.setProps({ ...props, hit: props.hits[1] }); findTestSubject(component, 'docTableDetailsFlyout').simulate('keydown', { key: 'ArrowLeft' }); - expect(props.setExpandedDoc).toHaveBeenCalledWith(expect.objectContaining({ _id: '1' })); + expect(props.setExpandedDoc).toHaveBeenCalledWith(expect.objectContaining({ id: 'i::1::' })); }); it('should not navigate with keypresses when already at the border of documents', () => { diff --git a/src/plugins/discover/public/components/discover_grid/discover_grid_flyout.tsx b/src/plugins/discover/public/components/discover_grid/discover_grid_flyout.tsx index a34e6da654699..c2165fc27ee2a 100644 --- a/src/plugins/discover/public/components/discover_grid/discover_grid_flyout.tsx +++ b/src/plugins/discover/public/components/discover_grid/discover_grid_flyout.tsx @@ -28,31 +28,24 @@ import { import { DocViewer } from '../../services/doc_views/components/doc_viewer/doc_viewer'; import { DocViewFilterFn } from '../../services/doc_views/doc_views_types'; import { useNavigationProps } from '../../hooks/use_navigation_props'; -import { ElasticSearchHit } from '../../types'; import { useDiscoverServices } from '../../hooks/use_discover_services'; +import type { DataTableRecord } from '../../types'; export interface DiscoverGridFlyoutProps { columns: string[]; - hit: ElasticSearchHit; - hits?: ElasticSearchHit[]; + hit: DataTableRecord; + hits?: DataTableRecord[]; indexPattern: DataView; onAddColumn: (column: string) => void; onClose: () => void; onFilter: DocViewFilterFn; onRemoveColumn: (column: string) => void; - setExpandedDoc: (doc: ElasticSearchHit) => void; + setExpandedDoc: (doc: DataTableRecord) => void; } -type ElasticSearchHitWithRouting = ElasticSearchHit & { _routing?: string }; - -function getDocFingerprintId(doc: ElasticSearchHitWithRouting) { - const routing = doc._routing || ''; - return [doc._index, doc._id, routing].join('||'); -} - -function getIndexByDocId(hits: ElasticSearchHit[], id: string) { +function getIndexByDocId(hits: DataTableRecord[], id: string) { return hits.findIndex((h) => { - return getDocFingerprintId(h) === id; + return h.id === id; }); } /** @@ -71,13 +64,10 @@ export function DiscoverGridFlyout({ }: DiscoverGridFlyoutProps) { const services = useDiscoverServices(); // Get actual hit with updated highlighted searches - const actualHit = useMemo( - () => hits?.find(({ _id, _index }) => hit._index === _index && _id === hit?._id) || hit, - [hit, hits] - ); + const actualHit = useMemo(() => hits?.find(({ id }) => id === hit?.id) || hit, [hit, hits]); const pageCount = useMemo(() => (hits ? hits.length : 0), [hits]); const activePage = useMemo(() => { - const id = getDocFingerprintId(hit); + const id = hit.id; if (!hits || pageCount <= 1) { return -1; } @@ -107,8 +97,8 @@ export function DiscoverGridFlyout({ const { singleDocProps, surrDocsProps } = useNavigationProps({ indexPatternId: indexPattern.id!, - rowIndex: hit._index, - rowId: hit._id, + rowIndex: hit.raw._index, + rowId: hit.raw._id, filterManager: services.filterManager, addBasePath: services.addBasePath, columns, diff --git a/src/plugins/discover/public/components/discover_grid/get_render_cell_value.test.tsx b/src/plugins/discover/public/components/discover_grid/get_render_cell_value.test.tsx index c706892c9f706..b797c8a969831 100644 --- a/src/plugins/discover/public/components/discover_grid/get_render_cell_value.test.tsx +++ b/src/plugins/discover/public/components/discover_grid/get_render_cell_value.test.tsx @@ -12,9 +12,9 @@ import { findTestSubject } from '@elastic/eui/lib/test'; import { mountWithIntl } from '@kbn/test-jest-helpers'; import { getRenderCellValueFn } from './get_render_cell_value'; import { indexPatternMock } from '../../__mocks__/index_pattern'; -import { flattenHit } from '@kbn/data-plugin/public'; -import { ElasticSearchHit } from '../../types'; import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; +import { buildDataTableRecord } from '../../utils/build_data_record'; +import { EsHitRecord } from '../../types'; const mockServices = { uiSettings: { @@ -33,7 +33,7 @@ jest.mock('../../hooks/use_discover_services', () => { }; }); -const rowsSource: ElasticSearchHit[] = [ +const rowsSource: EsHitRecord[] = [ { _id: '1', _index: 'test', @@ -45,7 +45,7 @@ const rowsSource: ElasticSearchHit[] = [ }, ]; -const rowsFields: ElasticSearchHit[] = [ +const rowsFields: EsHitRecord[] = [ { _id: '1', _index: 'test', @@ -58,7 +58,7 @@ const rowsFields: ElasticSearchHit[] = [ }, ]; -const rowsFieldsWithTopLevelObject: ElasticSearchHit[] = [ +const rowsFieldsWithTopLevelObject: EsHitRecord[] = [ { _id: '1', _index: 'test', @@ -71,16 +71,13 @@ const rowsFieldsWithTopLevelObject: ElasticSearchHit[] = [ }, ]; -const flatten = (hit: ElasticSearchHit): Record => { - return flattenHit(hit, indexPatternMock); -}; +const build = (hit: EsHitRecord) => buildDataTableRecord(hit, indexPatternMock); describe('Discover grid cell rendering', function () { it('renders bytes column correctly', () => { const DiscoverGridCellValue = getRenderCellValueFn( indexPatternMock, - rowsSource, - rowsSource.map(flatten), + rowsSource.map(build), false, [], 100, @@ -105,8 +102,7 @@ describe('Discover grid cell rendering', function () { it('renders bytes column correctly using _source when details is true', () => { const DiscoverGridCellValue = getRenderCellValueFn( indexPatternMock, - rowsSource, - rowsSource.map(flatten), + rowsSource.map(build), false, [], 100, @@ -132,8 +128,7 @@ describe('Discover grid cell rendering', function () { const closePopoverMockFn = jest.fn(); const DiscoverGridCellValue = getRenderCellValueFn( indexPatternMock, - rowsFields, - rowsFields.map(flatten), + rowsFields.map(build), false, [], 100, @@ -160,8 +155,7 @@ describe('Discover grid cell rendering', function () { it('renders _source column correctly', () => { const DiscoverGridCellValue = getRenderCellValueFn( indexPatternMock, - rowsSource, - rowsSource.map(flatten), + rowsSource.map(build), false, ['extension', 'bytes'], 100, @@ -235,8 +229,7 @@ describe('Discover grid cell rendering', function () { it('renders _source column correctly when isDetails is set to true', () => { const DiscoverGridCellValue = getRenderCellValueFn( indexPatternMock, - rowsSource, - rowsSource.map(flatten), + rowsSource.map(build), false, [], 100, @@ -309,8 +302,7 @@ describe('Discover grid cell rendering', function () { it('renders fields-based column correctly', () => { const DiscoverGridCellValue = getRenderCellValueFn( indexPatternMock, - rowsFields, - rowsFields.map(flatten), + rowsFields.map(build), true, ['extension', 'bytes'], 100, @@ -388,8 +380,7 @@ describe('Discover grid cell rendering', function () { it('limits amount of rendered items', () => { const DiscoverGridCellValue = getRenderCellValueFn( indexPatternMock, - rowsFields, - rowsFields.map(flatten), + rowsFields.map(build), true, ['extension', 'bytes'], // this is the number of rendered items @@ -468,8 +459,7 @@ describe('Discover grid cell rendering', function () { it('renders fields-based column correctly when isDetails is set to true', () => { const DiscoverGridCellValue = getRenderCellValueFn( indexPatternMock, - rowsFields, - rowsFields.map(flatten), + rowsFields.map(build), true, [], 100, @@ -547,8 +537,7 @@ describe('Discover grid cell rendering', function () { it('collect object fields and renders them like _source', () => { const DiscoverGridCellValue = getRenderCellValueFn( indexPatternMock, - rowsFieldsWithTopLevelObject, - rowsFieldsWithTopLevelObject.map(flatten), + rowsFieldsWithTopLevelObject.map(build), true, ['object.value', 'extension', 'bytes'], 100, @@ -590,8 +579,7 @@ describe('Discover grid cell rendering', function () { (indexPatternMock.getFieldByName as jest.Mock).mockReturnValueOnce(undefined); const DiscoverGridCellValue = getRenderCellValueFn( indexPatternMock, - rowsFieldsWithTopLevelObject, - rowsFieldsWithTopLevelObject.map(flatten), + rowsFieldsWithTopLevelObject.map(build), true, ['extension', 'bytes', 'object.value'], 100, @@ -633,8 +621,7 @@ describe('Discover grid cell rendering', function () { const closePopoverMockFn = jest.fn(); const DiscoverGridCellValue = getRenderCellValueFn( indexPatternMock, - rowsFieldsWithTopLevelObject, - rowsFieldsWithTopLevelObject.map(flatten), + rowsFieldsWithTopLevelObject.map(build), true, [], 100, @@ -699,8 +686,7 @@ describe('Discover grid cell rendering', function () { const closePopoverMockFn = jest.fn(); const DiscoverGridCellValue = getRenderCellValueFn( indexPatternMock, - rowsFieldsWithTopLevelObject, - rowsFieldsWithTopLevelObject.map(flatten), + rowsFieldsWithTopLevelObject.map(build), true, [], 100, @@ -728,8 +714,7 @@ describe('Discover grid cell rendering', function () { (indexPatternMock.getFieldByName as jest.Mock).mockReturnValueOnce(undefined); const DiscoverGridCellValue = getRenderCellValueFn( indexPatternMock, - rowsFieldsWithTopLevelObject, - rowsFieldsWithTopLevelObject.map(flatten), + rowsFieldsWithTopLevelObject.map(build), true, [], 100, @@ -763,8 +748,7 @@ describe('Discover grid cell rendering', function () { it('renders correctly when invalid row is given', () => { const DiscoverGridCellValue = getRenderCellValueFn( indexPatternMock, - rowsSource, - rowsSource.map(flatten), + rowsSource.map(build), false, [], 100, @@ -789,8 +773,7 @@ describe('Discover grid cell rendering', function () { it('renders correctly when invalid column is given', () => { const DiscoverGridCellValue = getRenderCellValueFn( indexPatternMock, - rowsSource, - rowsSource.map(flatten), + rowsSource.map(build), false, [], 100, @@ -814,7 +797,7 @@ describe('Discover grid cell rendering', function () { it('renders unmapped fields correctly', () => { (indexPatternMock.getFieldByName as jest.Mock).mockReturnValueOnce(undefined); - const rowsFieldsUnmapped: ElasticSearchHit[] = [ + const rowsFieldsUnmapped: EsHitRecord[] = [ { _id: '1', _index: 'test', @@ -828,8 +811,7 @@ describe('Discover grid cell rendering', function () { ]; const DiscoverGridCellValue = getRenderCellValueFn( indexPatternMock, - rowsFieldsUnmapped, - rowsFieldsUnmapped.map(flatten), + rowsFieldsUnmapped.map(build), true, ['unmapped'], 100, diff --git a/src/plugins/discover/public/components/discover_grid/get_render_cell_value.tsx b/src/plugins/discover/public/components/discover_grid/get_render_cell_value.tsx index b129b57dbec9c..5636e31efff5c 100644 --- a/src/plugins/discover/public/components/discover_grid/get_render_cell_value.tsx +++ b/src/plugins/discover/public/components/discover_grid/get_render_cell_value.tsx @@ -24,10 +24,9 @@ import { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; import { DiscoverGridContext } from './discover_grid_context'; import { JsonCodeEditor } from '../json_code_editor/json_code_editor'; import { defaultMonacoEditorWidth } from './constants'; -import { EsHitRecord } from '../../application/types'; import { formatFieldValue } from '../../utils/format_value'; import { formatHit } from '../../utils/format_hit'; -import { ElasticSearchHit, HitsFlattened } from '../../types'; +import { DataTableRecord, EsHitRecord } from '../../types'; import { useDiscoverServices } from '../../hooks/use_discover_services'; import { MAX_DOC_FIELDS_DISPLAYED } from '../../../common'; @@ -36,8 +35,7 @@ const CELL_CLASS = 'dscDiscoverGrid__cellValue'; export const getRenderCellValueFn = ( dataView: DataView, - rows: ElasticSearchHit[] | undefined, - rowsFlattened: HitsFlattened, + rows: DataTableRecord[] | undefined, useNewFieldsApi: boolean, fieldsToShow: string[], maxDocFieldsDisplayed: number, @@ -49,24 +47,16 @@ export const getRenderCellValueFn = const maxEntries = useMemo(() => uiSettings.get(MAX_DOC_FIELDS_DISPLAYED), [uiSettings]); const row = rows ? rows[rowIndex] : undefined; - const rowFlattened = rowsFlattened - ? (rowsFlattened[rowIndex] as Record) - : undefined; const field = dataView.fields.getByName(columnId); const ctx = useContext(DiscoverGridContext); useEffect(() => { - if ((row as EsHitRecord).isAnchor) { + if (row?.isAnchor) { setCellProps({ className: 'dscDocsGrid__cell--highlight', }); - } else if ( - ctx.expanded && - row && - ctx.expanded._id === row._id && - ctx.expanded._index === row._index - ) { + } else if (ctx.expanded && row && ctx.expanded.id === row.id) { setCellProps({ style: { backgroundColor: ctx.isDarkMode @@ -79,7 +69,7 @@ export const getRenderCellValueFn = } }, [ctx, row, setCellProps]); - if (typeof row === 'undefined' || typeof rowFlattened === 'undefined') { + if (typeof row === 'undefined') { return -; } @@ -90,14 +80,13 @@ export const getRenderCellValueFn = const useTopLevelObjectColumns = Boolean( useNewFieldsApi && !field && - row?.fields && - !(row.fields as Record)[columnId] + row?.raw.fields && + !(row.raw.fields as Record)[columnId] ); if (isDetails) { return renderPopoverContent({ - rowRaw: row, - rowFlattened, + row, field, columnId, dataView, @@ -109,7 +98,7 @@ export const getRenderCellValueFn = if (field?.type === '_source' || useTopLevelObjectColumns) { const pairs = useTopLevelObjectColumns - ? getTopLevelObjectPairs(row, columnId, dataView, fieldsToShow).slice( + ? getTopLevelObjectPairs(row.raw, columnId, dataView, fieldsToShow).slice( 0, maxDocFieldsDisplayed ) @@ -140,7 +129,7 @@ export const getRenderCellValueFn = // formatFieldValue guarantees sanitized values // eslint-disable-next-line react/no-danger dangerouslySetInnerHTML={{ - __html: formatFieldValue(rowFlattened[columnId], row, fieldFormats, dataView, field), + __html: formatFieldValue(row.flattened[columnId], row.raw, fieldFormats, dataView, field), }} /> ); @@ -158,10 +147,10 @@ function getInnerColumns(fields: Record, columnId: string) { ); } -function getJSON(columnId: string, rowRaw: ElasticSearchHit, useTopLevelObjectColumns: boolean) { +function getJSON(columnId: string, row: DataTableRecord, useTopLevelObjectColumns: boolean) { const json = useTopLevelObjectColumns - ? getInnerColumns(rowRaw.fields as Record, columnId) - : rowRaw; + ? getInnerColumns(row.raw.fields as Record, columnId) + : row.raw; return json as Record; } @@ -169,8 +158,7 @@ function getJSON(columnId: string, rowRaw: ElasticSearchHit, useTopLevelObjectCo * Helper function for the cell popover */ function renderPopoverContent({ - rowRaw, - rowFlattened, + row, field, columnId, dataView, @@ -178,8 +166,7 @@ function renderPopoverContent({ fieldFormats, closePopover, }: { - rowRaw: ElasticSearchHit; - rowFlattened: Record; + row: DataTableRecord; field: DataViewField | undefined; columnId: string; dataView: DataView; @@ -209,7 +196,7 @@ function renderPopoverContent({ @@ -226,7 +213,13 @@ function renderPopoverContent({ // formatFieldValue guarantees sanitized values // eslint-disable-next-line react/no-danger dangerouslySetInnerHTML={{ - __html: formatFieldValue(rowFlattened[columnId], rowRaw, fieldFormats, dataView, field), + __html: formatFieldValue( + row.flattened[columnId], + row.raw, + fieldFormats, + dataView, + field + ), }} /> @@ -239,7 +232,7 @@ function renderPopoverContent({ * this is used for legacy stuff like displaying products of our ecommerce dataset */ function getTopLevelObjectPairs( - row: ElasticSearchHit, + row: EsHitRecord, columnId: string, dataView: DataView, fieldsToShow: string[] diff --git a/src/plugins/discover/public/components/doc_table/components/table_row.test.tsx b/src/plugins/discover/public/components/doc_table/components/table_row.test.tsx index 03a12428569f7..1101c01784e84 100644 --- a/src/plugins/discover/public/components/doc_table/components/table_row.test.tsx +++ b/src/plugins/discover/public/components/doc_table/components/table_row.test.tsx @@ -17,6 +17,8 @@ import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; import { discoverServiceMock } from '../../../__mocks__/services'; import { DOC_HIDE_TIME_COLUMN_SETTING, MAX_DOC_FIELDS_DISPLAYED } from '../../../../common'; +import { buildDataTableRecord } from '../../../utils/build_data_record'; +import { EsHitRecord } from '../../../types'; jest.mock('../utils/row_formatter', () => { const originalModule = jest.requireActual('../utils/row_formatter'); @@ -64,7 +66,7 @@ const mockHit = { }, ], _source: { message: 'mock_message', bytes: 20 }, -}; +} as unknown as EsHitRecord; const mockFilterManager = createFilterManagerMock(); @@ -74,7 +76,7 @@ describe('Doc table row component', () => { columns: ['_source'], filter: mockInlineFilter, indexPattern: indexPatternWithTimefieldMock, - row: mockHit, + row: buildDataTableRecord(mockHit, indexPatternWithTimefieldMock), useNewFieldsApi: true, filterManager: mockFilterManager, addBasePath: (path: string) => path, diff --git a/src/plugins/discover/public/components/doc_table/components/table_row.tsx b/src/plugins/discover/public/components/doc_table/components/table_row.tsx index e08843006020a..898cb25417403 100644 --- a/src/plugins/discover/public/components/doc_table/components/table_row.tsx +++ b/src/plugins/discover/public/components/doc_table/components/table_row.tsx @@ -10,7 +10,6 @@ import React, { Fragment, useCallback, useMemo, useState } from 'react'; import classNames from 'classnames'; import { i18n } from '@kbn/i18n'; import { EuiButtonEmpty, EuiIcon } from '@elastic/eui'; -import { flattenHit } from '@kbn/data-plugin/public'; import { DataView } from '@kbn/data-views-plugin/public'; import { formatFieldValue } from '../../../utils/format_value'; import { DocViewer } from '../../../services/doc_views/components/doc_viewer'; @@ -18,19 +17,19 @@ import { TableCell } from './table_row/table_cell'; import { formatRow, formatTopLevelObject } from '../utils/row_formatter'; import { useNavigationProps } from '../../../hooks/use_navigation_props'; import { DocViewFilterFn } from '../../../services/doc_views/doc_views_types'; -import { ElasticSearchHit } from '../../../types'; +import { DataTableRecord, EsHitRecord } from '../../../types'; import { TableRowDetails } from './table_row_details'; import { useDiscoverServices } from '../../../hooks/use_discover_services'; import { DOC_HIDE_TIME_COLUMN_SETTING, MAX_DOC_FIELDS_DISPLAYED } from '../../../../common'; -export type DocTableRow = ElasticSearchHit & { +export type DocTableRow = EsHitRecord & { isAnchor?: boolean; }; export interface TableRowProps { columns: string[]; filter: DocViewFilterFn; - row: DocTableRow; + row: DataTableRecord; indexPattern: DataView; useNewFieldsApi: boolean; fieldsToShow: string[]; @@ -62,10 +61,6 @@ export const TableRow = ({ }); const anchorDocTableRowSubj = row.isAnchor ? ' docTableAnchorRow' : ''; - const flattenedRow = useMemo( - () => flattenHit(row, indexPattern, { includeIgnoredValues: true }), - [indexPattern, row] - ); const mapping = useMemo(() => indexPattern.fields.getByName, [indexPattern]); // toggle display of the rows details, a full list of the fields from each row @@ -82,8 +77,8 @@ export const TableRow = ({ } const formattedField = formatFieldValue( - flattenedRow[fieldName], - row, + row.flattened[fieldName], + row.raw, fieldFormats, indexPattern, mapping(fieldName) @@ -98,15 +93,15 @@ export const TableRow = ({ const inlineFilter = useCallback( (column: string, type: '+' | '-') => { const field = indexPattern.fields.getByName(column); - filter(field!, flattenedRow[column], type); + filter(field!, row.flattened, type); }, - [filter, flattenedRow, indexPattern.fields] + [filter, indexPattern.fields, row.flattened] ); const { singleDocProps, surrDocsProps } = useNavigationProps({ indexPatternId: indexPattern.id!, - rowIndex: row._index, - rowId: row._id, + rowIndex: row.raw._index, + rowId: row.raw._id, filterManager, addBasePath, columns, @@ -161,9 +156,9 @@ export const TableRow = ({ ); } else { columns.forEach(function (column: string) { - if (useNewFieldsApi && !mapping(column) && row.fields && !row.fields[column]) { + if (useNewFieldsApi && !mapping(column) && row.raw.fields && !row.raw.fields[column]) { const innerColumns = Object.fromEntries( - Object.entries(row.fields).filter(([key]) => { + Object.entries(row.raw.fields).filter(([key]) => { return key.indexOf(`${column}.`) === 0; }) ); @@ -185,7 +180,7 @@ export const TableRow = ({ // We should improve this and show a helpful tooltip why the filter buttons are not // there/disabled when there are ignored values. const isFilterable = Boolean( - mapping(column)?.filterable && filter && !row._ignored?.includes(column) + mapping(column)?.filterable && filter && !row.raw._ignored?.includes(column) ); rowCells.push( { - const mountComponent = (rows?: DocTableRow[]) => { + const mountComponent = (rows?: EsHitRecord[]) => { const props = { columns: ['_source'], indexPattern: indexPatternMock, - rows: rows || [ - { - _index: 'mock_index', - _id: '1', - _score: 1, - fields: [ - { - timestamp: '2020-20-01T12:12:12.123', - }, - ], - _source: { message: 'mock_message', bytes: 20 }, - }, - ], + rows: ( + rows || [ + { + _index: 'mock_index', + _id: '1', + _score: 1, + fields: [ + { + timestamp: '2020-20-01T12:12:12.123', + }, + ], + _source: { message: 'mock_message', bytes: 20 }, + } as EsHitRecord, + ] + ).map((row) => buildDataTableRecord(row, indexPatternMock)), + sort: [['order_date', 'desc']], isLoading: false, searchDescription: '', diff --git a/src/plugins/discover/public/components/doc_table/doc_table_wrapper.tsx b/src/plugins/discover/public/components/doc_table/doc_table_wrapper.tsx index 3562337961417..e226c859fb0a1 100644 --- a/src/plugins/discover/public/components/doc_table/doc_table_wrapper.tsx +++ b/src/plugins/discover/public/components/doc_table/doc_table_wrapper.tsx @@ -13,16 +13,17 @@ import { FormattedMessage } from '@kbn/i18n-react'; import { TableHeader } from './components/table_header/table_header'; import { SHOW_MULTIFIELDS } from '../../../common'; import { SortOrder } from './components/table_header/helpers'; -import { DocTableRow, TableRow } from './components/table_row'; +import { TableRow } from './components/table_row'; import { DocViewFilterFn } from '../../services/doc_views/doc_views_types'; import { getFieldsToShow } from '../../utils/get_fields_to_show'; import { useDiscoverServices } from '../../hooks/use_discover_services'; +import type { DataTableRecord } from '../../types'; export interface DocTableProps { /** * Rows of classic table */ - rows: DocTableRow[]; + rows: DataTableRecord[]; /** * Columns of classic table */ @@ -79,8 +80,8 @@ export interface DocTableProps { export interface DocTableRenderProps { columnLength: number; - rows: DocTableRow[]; - renderRows: (row: DocTableRow[]) => JSX.Element[]; + rows: DataTableRecord[]; + renderRows: (row: DataTableRecord[]) => JSX.Element[]; renderHeader: () => JSX.Element; onSkipBottomButtonClick: () => void; } @@ -155,10 +156,10 @@ export const DocTableWrapper = forwardRef( ); const renderRows = useCallback( - (rowsToRender: DocTableRow[]) => { + (rowsToRender: DataTableRecord[]) => { return rowsToRender.map((current) => ( { - const hit = { - _id: 'a', - _index: 'foo', - _type: 'doc', - _score: 1, - _source: { - foo: 'bar', - number: 42, - hello: '

World

', - also: 'with "quotes" or \'single quotes\'', - }, - }; let services: DiscoverServices; const createIndexPattern = () => { @@ -45,6 +34,18 @@ describe('Row formatter', () => { }; const indexPattern = createIndexPattern(); + const rawHit = { + _id: 'a', + _index: 'foo', + _score: 1, + _source: { + foo: 'bar', + number: 42, + hello: '

World

', + also: 'with "quotes" or \'single quotes\'', + }, + }; + const hit = buildDataTableRecord(rawHit, indexPattern); const fieldsToShow = indexPattern.fields.getAll().map((fld) => fld.name); @@ -138,15 +139,13 @@ describe('Row formatter', () => { }); it('formats document with highlighted fields first', () => { - expect( - formatRow( - { ...hit, highlight: { number: ['42'] } }, - indexPattern, - fieldsToShow, - 100, - services.fieldFormats - ) - ).toMatchInlineSnapshot(` + const highLightHit = buildDataTableRecord( + { ...rawHit, highlight: { number: ['42'] } }, + indexPattern + ); + + expect(formatRow(highLightHit, indexPattern, fieldsToShow, 100, services.fieldFormats)) + .toMatchInlineSnapshot(` { }; export const formatRow = ( - hit: estypes.SearchHit, + hit: DataTableRecord, indexPattern: DataView, fieldsToShow: string[], maxEntries: number, @@ -65,7 +65,7 @@ export const formatTopLevelObject = ( const displayKey = fields.getByName ? fields.getByName(key)?.displayName : undefined; const formatter = field ? indexPattern.getFormatterForField(field) - : { convert: (v: unknown, ...rest: unknown[]) => String(v) }; + : { convert: (v: unknown, ..._: unknown[]) => String(v) }; if (!values.map) return; const formatted = values .map((val: unknown) => diff --git a/src/plugins/discover/public/embeddable/saved_search_embeddable.tsx b/src/plugins/discover/public/embeddable/saved_search_embeddable.tsx index bc208aaaab0ec..cc6557e50e668 100644 --- a/src/plugins/discover/public/embeddable/saved_search_embeddable.tsx +++ b/src/plugins/discover/public/embeddable/saved_search_embeddable.tsx @@ -27,6 +27,8 @@ import { ISearchSource } from '@kbn/data-plugin/public'; import { DataView, DataViewField } from '@kbn/data-views-plugin/public'; import { UiActionsStart } from '@kbn/ui-actions-plugin/public'; import { KibanaContextProvider, KibanaThemeProvider } from '@kbn/kibana-react-plugin/public'; +import { buildDataTableRecord } from '../utils/build_data_record'; +import { DataTableRecord } from '../types'; import { ISearchEmbeddable, SearchInput, SearchOutput } from './types'; import { SavedSearch } from '../services/saved_searches'; import { SEARCH_EMBEDDABLE_TYPE } from './constants'; @@ -50,7 +52,6 @@ import { SortOrder } from '../components/doc_table/components/table_header/helpe import { VIEW_MODE } from '../components/view_mode_toggle'; import { updateSearchSource } from './utils/update_search_source'; import { FieldStatisticsTable } from '../application/main/components/field_stats_table'; -import { ElasticSearchHit } from '../types'; export type SearchProps = Partial & Partial & { @@ -59,9 +60,8 @@ export type SearchProps = Partial & sharedItemTitle?: string; inspectorAdapters?: Adapters; services: DiscoverServices; - filter?: (field: DataViewField, value: string[], operator: string) => void; - hits?: ElasticSearchHit[]; + hits?: DataTableRecord[]; totalHitCount?: number; onMoveColumn?: (column: string, index: number) => void; onUpdateRowHeight?: (rowHeight?: number) => void; @@ -210,7 +210,9 @@ export class SavedSearchEmbeddable ); this.updateOutput({ loading: false, error: undefined }); - this.searchProps!.rows = resp.hits.hits; + this.searchProps!.rows = resp.hits.hits.map((hit) => + buildDataTableRecord(hit, this.searchProps!.indexPattern) + ); this.searchProps!.totalHitCount = resp.hits.total as number; this.searchProps!.isLoading = false; } catch (error) { diff --git a/src/plugins/discover/public/embeddable/saved_search_grid.tsx b/src/plugins/discover/public/embeddable/saved_search_grid.tsx index ff72ed378aff3..7c567dc75dfa3 100644 --- a/src/plugins/discover/public/embeddable/saved_search_grid.tsx +++ b/src/plugins/discover/public/embeddable/saved_search_grid.tsx @@ -7,9 +7,9 @@ */ import React, { useState, memo } from 'react'; import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import { DataTableRecord } from '../types'; import { DiscoverGrid, DiscoverGridProps } from '../components/discover_grid/discover_grid'; import { TotalDocuments } from '../application/main/components/total_documents/total_documents'; -import { ElasticSearchHit } from '../types'; export interface DiscoverGridEmbeddableProps extends DiscoverGridProps { totalHitCount: number; @@ -18,7 +18,7 @@ export interface DiscoverGridEmbeddableProps extends DiscoverGridProps { export const DataGridMemoized = memo(DiscoverGrid); export function DiscoverGridEmbeddable(props: DiscoverGridEmbeddableProps) { - const [expandedDoc, setExpandedDoc] = useState(undefined); + const [expandedDoc, setExpandedDoc] = useState(undefined); return ( helper / hook', () => { getComputedFields: () => [], }; - const record = { test: 1 }; + const record = { _id: '1', _index: 't', test: 1 }; const props = { id: '1', @@ -233,6 +234,9 @@ describe('Test of helper / hook', () => { await hook.waitForNextUpdate(); }); - expect(hook.result.current.slice(0, 2)).toEqual([ElasticRequestState.Found, record]); + expect(hook.result.current.slice(0, 2)).toEqual([ + ElasticRequestState.Found, + buildDataTableRecord(record), + ]); }); }); diff --git a/src/plugins/discover/public/hooks/use_es_doc_search.ts b/src/plugins/discover/public/hooks/use_es_doc_search.ts index 84e759962de04..06c50af8e3889 100644 --- a/src/plugins/discover/public/hooks/use_es_doc_search.ts +++ b/src/plugins/discover/public/hooks/use_es_doc_search.ts @@ -10,10 +10,11 @@ import { useCallback, useEffect, useMemo, useState } from 'react'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { lastValueFrom } from 'rxjs'; import { DataView } from '@kbn/data-views-plugin/public'; +import { buildDataTableRecord } from '../utils/build_data_record'; +import { DataTableRecord } from '../types'; import { DocProps } from '../application/doc/components/doc'; import { ElasticRequestState } from '../application/doc/types'; import { SEARCH_FIELDS_FROM_SOURCE } from '../../common'; -import { ElasticSearchHit } from '../types'; import { useDiscoverServices } from './use_discover_services'; type RequestBody = Pick; @@ -26,9 +27,9 @@ export function useEsDocSearch({ index, indexPattern, requestSource, -}: DocProps): [ElasticRequestState, ElasticSearchHit | null, () => void] { +}: DocProps): [ElasticRequestState, DataTableRecord | null, () => void] { const [status, setStatus] = useState(ElasticRequestState.Loading); - const [hit, setHit] = useState(null); + const [hit, setHit] = useState(null); const { data, uiSettings } = useDiscoverServices(); const useNewFieldsApi = useMemo(() => !uiSettings.get(SEARCH_FIELDS_FROM_SOURCE), [uiSettings]); @@ -48,7 +49,7 @@ export function useEsDocSearch({ if (hits?.hits?.[0]) { setStatus(ElasticRequestState.Found); - setHit(hits.hits[0]); + setHit(buildDataTableRecord(hits.hits[0], indexPattern)); } else { setStatus(ElasticRequestState.NotFound); } diff --git a/src/plugins/discover/public/plugin.tsx b/src/plugins/discover/public/plugin.tsx index 24da9869238b0..e9eaba11d048c 100644 --- a/src/plugins/discover/public/plugin.tsx +++ b/src/plugins/discover/public/plugin.tsx @@ -241,8 +241,8 @@ export class DiscoverPlugin } > diff --git a/src/plugins/discover/public/services/doc_views/components/doc_viewer/doc_viewer.test.tsx b/src/plugins/discover/public/services/doc_views/components/doc_viewer/doc_viewer.test.tsx index 6d0018f695a33..41d483cb667c8 100644 --- a/src/plugins/discover/public/services/doc_views/components/doc_viewer/doc_viewer.test.tsx +++ b/src/plugins/discover/public/services/doc_views/components/doc_viewer/doc_viewer.test.tsx @@ -12,6 +12,7 @@ import { DocViewer } from './doc_viewer'; import { findTestSubject } from '@elastic/eui/lib/test'; import { getDocViewsRegistry } from '../../../../kibana_services'; import { DocViewRenderProps } from '../../doc_views_types'; +import { buildDataTableRecord } from '../../../../utils/build_data_record'; jest.mock('../../../../kibana_services', () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -75,7 +76,9 @@ test('Render with 1 tab displaying error message', () => { component: SomeComponent, }); - const renderProps = { hit: {} } as DocViewRenderProps; + const renderProps = { + hit: buildDataTableRecord({ _index: 't', _id: '1' }), + } as DocViewRenderProps; const errorMsg = 'Catch me if you can!'; const wrapper = mount(); diff --git a/src/plugins/discover/public/services/doc_views/components/doc_viewer/doc_viewer_tab.test.tsx b/src/plugins/discover/public/services/doc_views/components/doc_viewer/doc_viewer_tab.test.tsx index 1d5b1d4577652..af0bda4b67713 100644 --- a/src/plugins/discover/public/services/doc_views/components/doc_viewer/doc_viewer_tab.test.tsx +++ b/src/plugins/discover/public/services/doc_views/components/doc_viewer/doc_viewer_tab.test.tsx @@ -10,17 +10,18 @@ import React from 'react'; import { shallow } from 'enzyme'; import { DocViewerTab } from './doc_viewer_tab'; import { indexPatternMock } from '../../../../__mocks__/index_pattern'; -import { ElasticSearchHit } from '../../../../types'; +import { buildDataTableRecord } from '../../../../utils/build_data_record'; describe('DocViewerTab', () => { test('changing columns triggers an update', () => { + const hit = buildDataTableRecord({ _index: 'test', _id: '1' }, indexPatternMock); const props = { title: 'test', component: jest.fn(), id: 1, render: jest.fn(), renderProps: { - hit: {} as ElasticSearchHit, + hit, columns: ['test'], indexPattern: indexPatternMock, }, @@ -31,7 +32,7 @@ describe('DocViewerTab', () => { const nextProps = { ...props, renderProps: { - hit: {} as ElasticSearchHit, + hit, columns: ['test2'], indexPattern: indexPatternMock, }, diff --git a/src/plugins/discover/public/services/doc_views/components/doc_viewer/doc_viewer_tab.tsx b/src/plugins/discover/public/services/doc_views/components/doc_viewer/doc_viewer_tab.tsx index 0ad3187e1cf4f..837ee910176b9 100644 --- a/src/plugins/discover/public/services/doc_views/components/doc_viewer/doc_viewer_tab.tsx +++ b/src/plugins/discover/public/services/doc_views/components/doc_viewer/doc_viewer_tab.tsx @@ -42,9 +42,8 @@ export class DocViewerTab extends React.Component { shouldComponentUpdate(nextProps: Props, nextState: State) { return ( - nextProps.renderProps.hit._id !== this.props.renderProps.hit._id || - nextProps.renderProps.hit._index !== this.props.renderProps.hit._index || - !isEqual(nextProps.renderProps.hit.highlight, this.props.renderProps.hit.highlight) || + nextProps.renderProps.hit.id !== this.props.renderProps.hit.id || + !isEqual(nextProps.renderProps.hit.raw.highlight, this.props.renderProps.hit.raw.highlight) || nextProps.id !== this.props.id || !isEqual(nextProps.renderProps.columns, this.props.renderProps.columns) || nextState.hasError diff --git a/src/plugins/discover/public/services/doc_views/components/doc_viewer_source/__snapshots__/source.test.tsx.snap b/src/plugins/discover/public/services/doc_views/components/doc_viewer_source/__snapshots__/source.test.tsx.snap deleted file mode 100644 index c639e60dd9a7c..0000000000000 --- a/src/plugins/discover/public/services/doc_views/components/doc_viewer_source/__snapshots__/source.test.tsx.snap +++ /dev/null @@ -1,128 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`Source Viewer component renders error state 1`] = ` -
-
-
- -
-
-
-

- An Error Occurred -

-
-
-
- Could not fetch data at this time. Refresh the tab to try again. -
- -
-
-
-
-
-
-`; - -exports[`Source Viewer component renders json code editor 1`] = ` -
-
-
-
- - - -
-
-
-
-
-
-`; - -exports[`Source Viewer component renders loading state 1`] = ` -
- -
- Loading JSON -
-
-`; diff --git a/src/plugins/discover/public/services/doc_views/components/doc_viewer_source/source.test.tsx b/src/plugins/discover/public/services/doc_views/components/doc_viewer_source/source.test.tsx index 2f08caa4f6817..f51ddd13f1afe 100644 --- a/src/plugins/discover/public/services/doc_views/components/doc_viewer_source/source.test.tsx +++ b/src/plugins/discover/public/services/doc_views/components/doc_viewer_source/source.test.tsx @@ -15,6 +15,7 @@ import * as useUiSettingHook from '@kbn/kibana-react-plugin/public/ui_settings/u import { EuiButton, EuiEmptyPrompt, EuiLoadingSpinner } from '@elastic/eui'; import { JsonCodeEditorCommon } from '../../../../components/json_code_editor/json_code_editor_common'; import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; +import { buildDataTableRecord } from '../../../../utils/build_data_record'; const mockIndexPattern = { getComputedFields: () => [], @@ -51,7 +52,6 @@ describe('Source Viewer component', () => { /> ); - expect(comp.children().render()).toMatchSnapshot(); const loadingIndicator = comp.find(EuiLoadingSpinner); expect(loadingIndicator).not.toBe(null); }); @@ -70,7 +70,6 @@ describe('Source Viewer component', () => { /> ); - expect(comp.children().render()).toMatchSnapshot(); const errorPrompt = comp.find(EuiEmptyPrompt); expect(errorPrompt.length).toBe(1); const refreshButton = comp.find(EuiButton); @@ -78,9 +77,8 @@ describe('Source Viewer component', () => { }); test('renders json code editor', () => { - const mockHit = { + const mockHit = buildDataTableRecord({ _index: 'logstash-2014.09.09', - _type: 'doc', _id: 'id123', _score: 1, _source: { @@ -95,7 +93,7 @@ describe('Source Viewer component', () => { scripted: 123, _underscore: 123, }, - } as never; + }); jest.spyOn(hooks, 'useEsDocSearch').mockImplementation(() => [2, mockHit, () => {}]); jest.spyOn(useUiSettingHook, 'useUiSetting').mockImplementation(() => { return false; @@ -111,7 +109,6 @@ describe('Source Viewer component', () => { /> ); - expect(comp.children().render()).toMatchSnapshot(); const jsonCodeEditor = comp.find(JsonCodeEditorCommon); expect(jsonCodeEditor).not.toBe(null); }); diff --git a/src/plugins/discover/public/services/doc_views/components/doc_viewer_source/source.tsx b/src/plugins/discover/public/services/doc_views/components/doc_viewer_source/source.tsx index cbc14c9382e1c..ade467df853b0 100644 --- a/src/plugins/discover/public/services/doc_views/components/doc_viewer_source/source.tsx +++ b/src/plugins/discover/public/services/doc_views/components/doc_viewer_source/source.tsx @@ -56,7 +56,7 @@ export const DocViewerSource = ({ useEffect(() => { if (reqState === ElasticRequestState.Found && hit) { - setJsonValue(JSON.stringify(hit, undefined, 2)); + setJsonValue(JSON.stringify(hit.raw, undefined, 2)); } }, [reqState, hit]); diff --git a/src/plugins/discover/public/services/doc_views/components/doc_viewer_table/legacy/table.test.tsx b/src/plugins/discover/public/services/doc_views/components/doc_viewer_table/legacy/table.test.tsx index 0e75ab7d7800d..053af6643eb79 100644 --- a/src/plugins/discover/public/services/doc_views/components/doc_viewer_table/legacy/table.test.tsx +++ b/src/plugins/discover/public/services/doc_views/components/doc_viewer_table/legacy/table.test.tsx @@ -12,9 +12,9 @@ import { findTestSubject } from '@elastic/eui/lib/test'; import { DocViewerLegacyTable } from './table'; import { DataView } from '@kbn/data-views-plugin/public'; import { DocViewRenderProps } from '../../../doc_views_types'; -import { ElasticSearchHit } from '../../../../../types'; import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; import { DiscoverServices } from '../../../../../build_services'; +import { buildDataTableRecord } from '../../../../../utils/build_data_record'; const services = { uiSettings: { @@ -85,14 +85,14 @@ describe('DocViewTable at Discover', () => { // At Discover's main view, all buttons are rendered // check for existence of action buttons and warnings - const hit = { - _index: 'logstash-2014.09.09', - _type: 'doc', - _id: 'id123', - _score: 1, - _source: { - message: - 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit. \ + const hit = buildDataTableRecord( + { + _index: 'logstash-2014.09.09', + _id: 'id123', + _score: 1, + _source: { + message: + 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit. \ Aenean commodo ligula eget dolor. Aenean massa. Cum sociis natoque penatibus \ et magnis dis parturient montes, nascetur ridiculus mus. Donec quam felis, \ ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa quis enim. \ @@ -100,17 +100,19 @@ describe('DocViewTable at Discover', () => { rhoncus ut, imperdiet a, venenatis vitae, justo. Nullam dictum felis eu pede mollis pretium. \ Integer tincidunt. Cras dapibus. Vivamus elementum semper nisi. Aenean vulputate eleifend tellus. \ Phasellus ullamcorper ipsum rutrum nunc. Nunc nonummy metus. Vestibulum volutpat pretium libero. Cras id dui. Aenean ut', - extension: 'html', - not_mapped: 'yes', - bytes: 100, - objectArray: [{ foo: true }], - relatedContent: { - test: 1, + extension: 'html', + not_mapped: 'yes', + bytes: 100, + objectArray: [{ foo: true }], + relatedContent: { + test: 1, + }, + scripted: 123, + _underscore: 123, }, - scripted: 123, - _underscore: 123, }, - } as ElasticSearchHit; + indexPattern + ); const props = { hit, @@ -194,14 +196,14 @@ describe('DocViewTable at Discover', () => { describe('DocViewTable at Discover Context', () => { // here no toggleColumnButtons are rendered - const hit = { - _index: 'logstash-2014.09.09', - _type: 'doc', - _id: 'id123', - _score: 1, - _source: { - message: - 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit. \ + const hit = buildDataTableRecord( + { + _index: 'logstash-2014.09.09', + _id: 'id123', + _score: 1, + _source: { + message: + 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit. \ Aenean commodo ligula eget dolor. Aenean massa. Cum sociis natoque penatibus \ et magnis dis parturient montes, nascetur ridiculus mus. Donec quam felis, \ ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa quis enim. \ @@ -209,8 +211,10 @@ describe('DocViewTable at Discover Context', () => { rhoncus ut, imperdiet a, venenatis vitae, justo. Nullam dictum felis eu pede mollis pretium. \ Integer tincidunt. Cras dapibus. Vivamus elementum semper nisi. Aenean vulputate eleifend tellus. \ Phasellus ullamcorper ipsum rutrum nunc. Nunc nonummy metus. Vestibulum volutpat pretium libero. Cras id dui. Aenean ut', + }, }, - } as ElasticSearchHit; + indexPattern + ); const props = { hit, columns: ['extension'], @@ -246,16 +250,18 @@ describe('DocViewTable at Discover Context', () => { }); describe('DocViewTable at Discover Doc', () => { - const hit = { - _index: 'logstash-2014.09.09', - _score: 1, - _type: 'doc', - _id: 'id123', - _source: { - extension: 'html', - not_mapped: 'yes', + const hit = buildDataTableRecord( + { + _index: 'logstash-2014.09.09', + _score: 1, + _id: 'id123', + _source: { + extension: 'html', + not_mapped: 'yes', + }, }, - }; + indexPattern + ); // here no action buttons are rendered const props = { hit, @@ -370,20 +376,22 @@ describe('DocViewTable at Discover Doc with Fields API', () => { return indexPatterneCommerce.fields.getAll().find((field) => field.name === name); }; - const fieldsHit = { - _index: 'logstash-2014.09.09', - _type: 'doc', - _id: 'id123', - _score: 1.0, - fields: { - category: "Women's Clothing", - 'category.keyword': "Women's Clothing", - customer_first_name: 'Betty', - 'customer_first_name.keyword': 'Betty', - 'customer_first_name.nickname': 'Betsy', - 'city.raw': 'Los Angeles', + const fieldsHit = buildDataTableRecord( + { + _index: 'logstash-2014.09.09', + _id: 'id123', + _score: 1.0, + fields: { + category: "Women's Clothing", + 'category.keyword': "Women's Clothing", + customer_first_name: 'Betty', + 'customer_first_name.keyword': 'Betty', + 'customer_first_name.nickname': 'Betsy', + 'city.raw': 'Los Angeles', + }, }, - }; + indexPattern + ); const props = { hit: fieldsHit, columns: ['Document'], diff --git a/src/plugins/discover/public/services/doc_views/components/doc_viewer_table/legacy/table.tsx b/src/plugins/discover/public/services/doc_views/components/doc_viewer_table/legacy/table.tsx index 152a5451e7760..ad4345a79b166 100644 --- a/src/plugins/discover/public/services/doc_views/components/doc_viewer_table/legacy/table.tsx +++ b/src/plugins/discover/public/services/doc_views/components/doc_viewer_table/legacy/table.tsx @@ -9,7 +9,6 @@ import '../table.scss'; import React, { useCallback, useMemo } from 'react'; import { EuiInMemoryTable } from '@elastic/eui'; -import { flattenHit } from '@kbn/data-plugin/public'; import { getTypeForFieldIcon } from '../../../../../utils/get_type_for_field_icon'; import { useDiscoverServices } from '../../../../../hooks/use_discover_services'; import { SHOW_MULTIFIELDS } from '../../../../../../common'; @@ -57,10 +56,9 @@ export const DocViewerLegacyTable = ({ }; }, []); - const flattened = flattenHit(hit, dataView, { source: true, includeIgnoredValues: true }); - const fieldsToShow = getFieldsToShow(Object.keys(flattened), dataView, showMultiFields); + const fieldsToShow = getFieldsToShow(Object.keys(hit.flattened), dataView, showMultiFields); - const items: FieldRecordLegacy[] = Object.keys(flattened) + const items: FieldRecordLegacy[] = Object.keys(hit.flattened) .filter((fieldName) => { return fieldsToShow.includes(fieldName); }) @@ -79,13 +77,13 @@ export const DocViewerLegacyTable = ({ : fieldMapping ? getTypeForFieldIcon(fieldMapping) : undefined; - const ignored = getIgnoredReason(fieldMapping ?? field, hit._ignored); + const ignored = getIgnoredReason(fieldMapping ?? field, hit.raw._ignored); return { action: { onToggleColumn, onFilter: filter, isActive: !!columns?.includes(field), - flattenedField: flattened[field], + flattenedField: hit.flattened[field], }, field: { field, @@ -96,8 +94,8 @@ export const DocViewerLegacyTable = ({ }, value: { formattedValue: formatFieldValue( - flattened[field], - hit, + hit.flattened[field], + hit.raw, fieldFormats, dataView, fieldMapping diff --git a/src/plugins/discover/public/services/doc_views/components/doc_viewer_table/table.tsx b/src/plugins/discover/public/services/doc_views/components/doc_viewer_table/table.tsx index 48a869c86d3e8..7e61918aa7ae8 100644 --- a/src/plugins/discover/public/services/doc_views/components/doc_viewer_table/table.tsx +++ b/src/plugins/discover/public/services/doc_views/components/doc_viewer_table/table.tsx @@ -28,7 +28,6 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import { debounce } from 'lodash'; import { Storage } from '@kbn/kibana-utils-plugin/public'; -import { flattenHit } from '@kbn/data-plugin/public'; import { getTypeForFieldIcon } from '../../../../utils/get_type_for_field_icon'; import { useDiscoverServices } from '../../../../hooks/use_discover_services'; import { usePager } from '../../../../hooks/use_pager'; @@ -119,7 +118,7 @@ export const DocViewerTable = ({ getPinnedFields(currentDataViewId, storage) ); - const flattened = flattenHit(hit, dataView, { source: true, includeIgnoredValues: true }); + const flattened = hit.flattened; const fieldsToShow = getFieldsToShow(Object.keys(flattened), dataView, showMultiFields); const searchPlaceholder = i18n.translate('discover.docView.table.searchPlaceHolder', { @@ -164,7 +163,7 @@ export const DocViewerTable = ({ ? getTypeForFieldIcon(fieldMapping) : undefined; - const ignored = getIgnoredReason(fieldMapping ?? field, hit._ignored); + const ignored = getIgnoredReason(fieldMapping ?? field, hit.raw._ignored); return { action: { @@ -184,8 +183,8 @@ export const DocViewerTable = ({ }, value: { formattedValue: formatFieldValue( - flattened[field], - hit, + hit.flattened[field], + hit.raw, fieldFormats, dataView, fieldMapping diff --git a/src/plugins/discover/public/services/doc_views/doc_views_registry.ts b/src/plugins/discover/public/services/doc_views/doc_views_registry.ts index 8ee8741d73d3f..d6bf89c5fef70 100644 --- a/src/plugins/discover/public/services/doc_views/doc_views_registry.ts +++ b/src/plugins/discover/public/services/doc_views/doc_views_registry.ts @@ -6,8 +6,8 @@ * Side Public License, v 1. */ -import { ElasticSearchHit } from '../../types'; import { DocView, DocViewInput, DocViewInputFn } from './doc_views_types'; +import { DataTableRecord } from '../../types'; export class DocViewsRegistry { private docViews: DocView[] = []; @@ -25,7 +25,7 @@ export class DocViewsRegistry { /** * Returns a sorted array of doc_views for rendering tabs */ - getDocViewsSorted(hit: ElasticSearchHit) { + getDocViewsSorted(hit: DataTableRecord) { return this.docViews .filter((docView) => docView.shouldShow(hit)) .sort((a, b) => (Number(a.order) > Number(b.order) ? 1 : -1)); diff --git a/src/plugins/discover/public/services/doc_views/doc_views_types.ts b/src/plugins/discover/public/services/doc_views/doc_views_types.ts index 171b29a86bb05..ca05bce452f91 100644 --- a/src/plugins/discover/public/services/doc_views/doc_views_types.ts +++ b/src/plugins/discover/public/services/doc_views/doc_views_types.ts @@ -7,7 +7,7 @@ */ import { DataView, DataViewField } from '@kbn/data-views-plugin/public'; -import { ElasticSearchHit } from '../../types'; +import { DataTableRecord } from '../../types'; import { IgnoredReason } from '../../utils/get_ignored_reason'; export interface FieldMapping { @@ -26,7 +26,7 @@ export type DocViewFilterFn = ( ) => void; export interface DocViewRenderProps { - hit: ElasticSearchHit; + hit: DataTableRecord; indexPattern: DataView; columns?: string[]; filter?: DocViewFilterFn; @@ -41,7 +41,7 @@ export type DocViewRenderFn = ( export interface BaseDocViewInput { order: number; - shouldShow?: (hit: ElasticSearchHit) => boolean; + shouldShow?: (hit: DataTableRecord) => boolean; title: string; } diff --git a/src/plugins/discover/public/types.ts b/src/plugins/discover/public/types.ts index fa8582337349d..f96edccb5f9bf 100644 --- a/src/plugins/discover/public/types.ts +++ b/src/plugins/discover/public/types.ts @@ -6,14 +6,35 @@ * Side Public License, v 1. */ -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; - -export type ElasticSearchHit = estypes.SearchHit; - -export type HitsFlattened = Array>; +import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; export type ValueToStringConverter = ( rowIndex: number, columnId: string, options?: { disableMultiline?: boolean } ) => { formattedString: string; withFormula: boolean }; + +export interface EsHitRecord extends Omit { + _source?: Record; +} +/** + * This is the record/row of data provided to our Data Table + */ +export interface DataTableRecord { + /** + * A unique id generated by index, id and routing of a record + */ + id: string; + /** + * The document returned by Elasticsearch for search queries + */ + raw: EsHitRecord; + /** + * A flattened version of the ES doc or data provided by SQL, aggregations ... + */ + flattened: Record; + /** + * Determines that the given doc is the anchor doc when rendering view surrounding docs + */ + isAnchor?: boolean; +} diff --git a/src/plugins/discover/public/utils/build_data_record.ts b/src/plugins/discover/public/utils/build_data_record.ts new file mode 100644 index 0000000000000..2691fddfb66a9 --- /dev/null +++ b/src/plugins/discover/public/utils/build_data_record.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { DataView } from '@kbn/data-views-plugin/common'; +import { flattenHit } from '@kbn/data-plugin/common'; +import { getDocId } from './get_doc_id'; +import type { DataTableRecord, EsHitRecord } from '../types'; + +/** + * Build a record for data table, explorer + classic one + * @param doc the document returned from Elasticsearch + * @param dataView this current data view + * @param isAnchor determines if the given doc is the anchor doc when viewing surrounding documents + */ +export function buildDataTableRecord( + doc: EsHitRecord, + dataView?: DataView, + isAnchor?: boolean +): DataTableRecord { + return { + id: getDocId(doc), + raw: doc, + flattened: flattenHit(doc, dataView, { includeIgnoredValues: true }), + isAnchor, + }; +} + +/** + * Helper to build multiple DataTableRecords at once, saved a bit of testing code lines + * @param docs Array of documents returned from Elasticsearch + * @param dataView this current data view + */ +export function buildDataTableRecordList( + docs: EsHitRecord[], + dataView?: DataView +): DataTableRecord[] { + return docs.map((doc) => buildDataTableRecord(doc, dataView)); +} diff --git a/src/plugins/discover/public/utils/convert_value_to_string.test.tsx b/src/plugins/discover/public/utils/convert_value_to_string.test.tsx index e77d664851a23..64042c190feba 100644 --- a/src/plugins/discover/public/utils/convert_value_to_string.test.tsx +++ b/src/plugins/discover/public/utils/convert_value_to_string.test.tsx @@ -14,7 +14,6 @@ describe('convertValueToString', () => { it('should convert a keyword value to text', () => { const result = convertValueToString({ rows: discoverGridContextComplexMock.rows, - rowsFlattened: discoverGridContextComplexMock.rowsFlattened, dataView: discoverGridContextComplexMock.indexPattern, services: discoverServiceMock, columnId: 'keyword_key', @@ -30,7 +29,6 @@ describe('convertValueToString', () => { it('should convert a text value to text', () => { const result = convertValueToString({ rows: discoverGridContextComplexMock.rows, - rowsFlattened: discoverGridContextComplexMock.rowsFlattened, dataView: discoverGridContextComplexMock.indexPattern, services: discoverServiceMock, columnId: 'text_message', @@ -46,7 +44,6 @@ describe('convertValueToString', () => { it('should convert a multiline text value to text', () => { const result = convertValueToString({ rows: discoverGridContextComplexMock.rows, - rowsFlattened: discoverGridContextComplexMock.rowsFlattened, dataView: discoverGridContextComplexMock.indexPattern, services: discoverServiceMock, columnId: 'text_message', @@ -63,7 +60,6 @@ describe('convertValueToString', () => { it('should convert a number value to text', () => { const result = convertValueToString({ rows: discoverGridContextComplexMock.rows, - rowsFlattened: discoverGridContextComplexMock.rowsFlattened, dataView: discoverGridContextComplexMock.indexPattern, services: discoverServiceMock, columnId: 'number_price', @@ -79,7 +75,6 @@ describe('convertValueToString', () => { it('should convert a date value to text', () => { const result = convertValueToString({ rows: discoverGridContextComplexMock.rows, - rowsFlattened: discoverGridContextComplexMock.rowsFlattened, dataView: discoverGridContextComplexMock.indexPattern, services: discoverServiceMock, columnId: 'date', @@ -95,7 +90,6 @@ describe('convertValueToString', () => { it('should convert a date nanos value to text', () => { const result = convertValueToString({ rows: discoverGridContextComplexMock.rows, - rowsFlattened: discoverGridContextComplexMock.rowsFlattened, dataView: discoverGridContextComplexMock.indexPattern, services: discoverServiceMock, columnId: 'date_nanos', @@ -111,7 +105,6 @@ describe('convertValueToString', () => { it('should convert a boolean value to text', () => { const result = convertValueToString({ rows: discoverGridContextComplexMock.rows, - rowsFlattened: discoverGridContextComplexMock.rowsFlattened, dataView: discoverGridContextComplexMock.indexPattern, services: discoverServiceMock, columnId: 'bool_enabled', @@ -127,7 +120,6 @@ describe('convertValueToString', () => { it('should convert a binary value to text', () => { const result = convertValueToString({ rows: discoverGridContextComplexMock.rows, - rowsFlattened: discoverGridContextComplexMock.rowsFlattened, dataView: discoverGridContextComplexMock.indexPattern, services: discoverServiceMock, columnId: 'binary_blob', @@ -143,7 +135,6 @@ describe('convertValueToString', () => { it('should convert an object value to text', () => { const result = convertValueToString({ rows: discoverGridContextComplexMock.rows, - rowsFlattened: discoverGridContextComplexMock.rowsFlattened, dataView: discoverGridContextComplexMock.indexPattern, services: discoverServiceMock, columnId: 'object_user.first', @@ -159,7 +150,6 @@ describe('convertValueToString', () => { it('should convert a nested value to text', () => { const result = convertValueToString({ rows: discoverGridContextComplexMock.rows, - rowsFlattened: discoverGridContextComplexMock.rowsFlattened, dataView: discoverGridContextComplexMock.indexPattern, services: discoverServiceMock, columnId: 'nested_user', @@ -177,7 +167,6 @@ describe('convertValueToString', () => { it('should convert a flattened value to text', () => { const result = convertValueToString({ rows: discoverGridContextComplexMock.rows, - rowsFlattened: discoverGridContextComplexMock.rowsFlattened, dataView: discoverGridContextComplexMock.indexPattern, services: discoverServiceMock, columnId: 'flattened_labels', @@ -193,7 +182,6 @@ describe('convertValueToString', () => { it('should convert a range value to text', () => { const result = convertValueToString({ rows: discoverGridContextComplexMock.rows, - rowsFlattened: discoverGridContextComplexMock.rowsFlattened, dataView: discoverGridContextComplexMock.indexPattern, services: discoverServiceMock, columnId: 'range_time_frame', @@ -211,7 +199,6 @@ describe('convertValueToString', () => { it('should convert a rank features value to text', () => { const result = convertValueToString({ rows: discoverGridContextComplexMock.rows, - rowsFlattened: discoverGridContextComplexMock.rowsFlattened, dataView: discoverGridContextComplexMock.indexPattern, services: discoverServiceMock, columnId: 'rank_features', @@ -227,7 +214,6 @@ describe('convertValueToString', () => { it('should convert a histogram value to text', () => { const result = convertValueToString({ rows: discoverGridContextComplexMock.rows, - rowsFlattened: discoverGridContextComplexMock.rowsFlattened, dataView: discoverGridContextComplexMock.indexPattern, services: discoverServiceMock, columnId: 'histogram', @@ -243,7 +229,6 @@ describe('convertValueToString', () => { it('should convert a IP value to text', () => { const result = convertValueToString({ rows: discoverGridContextComplexMock.rows, - rowsFlattened: discoverGridContextComplexMock.rowsFlattened, dataView: discoverGridContextComplexMock.indexPattern, services: discoverServiceMock, columnId: 'ip_addr', @@ -259,7 +244,6 @@ describe('convertValueToString', () => { it('should convert a version value to text', () => { const result = convertValueToString({ rows: discoverGridContextComplexMock.rows, - rowsFlattened: discoverGridContextComplexMock.rowsFlattened, dataView: discoverGridContextComplexMock.indexPattern, services: discoverServiceMock, columnId: 'version', @@ -275,7 +259,6 @@ describe('convertValueToString', () => { it('should convert a vector value to text', () => { const result = convertValueToString({ rows: discoverGridContextComplexMock.rows, - rowsFlattened: discoverGridContextComplexMock.rowsFlattened, dataView: discoverGridContextComplexMock.indexPattern, services: discoverServiceMock, columnId: 'vector', @@ -291,7 +274,6 @@ describe('convertValueToString', () => { it('should convert a geo point value to text', () => { const result = convertValueToString({ rows: discoverGridContextComplexMock.rows, - rowsFlattened: discoverGridContextComplexMock.rowsFlattened, dataView: discoverGridContextComplexMock.indexPattern, services: discoverServiceMock, columnId: 'geo_point', @@ -307,7 +289,6 @@ describe('convertValueToString', () => { it('should convert a geo point object value to text', () => { const result = convertValueToString({ rows: discoverGridContextComplexMock.rows, - rowsFlattened: discoverGridContextComplexMock.rowsFlattened, dataView: discoverGridContextComplexMock.indexPattern, services: discoverServiceMock, columnId: 'geo_point', @@ -323,7 +304,6 @@ describe('convertValueToString', () => { it('should convert an array value to text', () => { const result = convertValueToString({ rows: discoverGridContextComplexMock.rows, - rowsFlattened: discoverGridContextComplexMock.rowsFlattened, dataView: discoverGridContextComplexMock.indexPattern, services: discoverServiceMock, columnId: 'array_tags', @@ -339,7 +319,6 @@ describe('convertValueToString', () => { it('should convert a shape value to text', () => { const result = convertValueToString({ rows: discoverGridContextComplexMock.rows, - rowsFlattened: discoverGridContextComplexMock.rowsFlattened, dataView: discoverGridContextComplexMock.indexPattern, services: discoverServiceMock, columnId: 'geometry', @@ -357,7 +336,6 @@ describe('convertValueToString', () => { it('should convert a runtime value to text', () => { const result = convertValueToString({ rows: discoverGridContextComplexMock.rows, - rowsFlattened: discoverGridContextComplexMock.rowsFlattened, dataView: discoverGridContextComplexMock.indexPattern, services: discoverServiceMock, columnId: 'runtime_number', @@ -373,7 +351,6 @@ describe('convertValueToString', () => { it('should convert a scripted value to text', () => { const result = convertValueToString({ rows: discoverGridContextComplexMock.rows, - rowsFlattened: discoverGridContextComplexMock.rowsFlattened, dataView: discoverGridContextComplexMock.indexPattern, services: discoverServiceMock, columnId: 'scripted_string', @@ -389,7 +366,6 @@ describe('convertValueToString', () => { it('should return an empty string and not fail', () => { const result = convertValueToString({ rows: discoverGridContextComplexMock.rows, - rowsFlattened: discoverGridContextComplexMock.rowsFlattened, dataView: discoverGridContextComplexMock.indexPattern, services: discoverServiceMock, columnId: 'unknown', @@ -405,7 +381,6 @@ describe('convertValueToString', () => { it('should return an empty string when rowIndex is out of range', () => { const result = convertValueToString({ rows: discoverGridContextComplexMock.rows, - rowsFlattened: discoverGridContextComplexMock.rowsFlattened, dataView: discoverGridContextComplexMock.indexPattern, services: discoverServiceMock, columnId: 'unknown', @@ -421,7 +396,6 @@ describe('convertValueToString', () => { it('should return _source value', () => { const result = convertValueToString({ rows: discoverGridContextMock.rows, - rowsFlattened: discoverGridContextMock.rowsFlattened, dataView: discoverGridContextMock.indexPattern, services: discoverServiceMock, columnId: '_source', @@ -445,7 +419,6 @@ describe('convertValueToString', () => { it('should return a formatted _source value', () => { const result = convertValueToString({ rows: discoverGridContextMock.rows, - rowsFlattened: discoverGridContextMock.rowsFlattened, dataView: discoverGridContextMock.indexPattern, services: discoverServiceMock, columnId: '_source', @@ -463,7 +436,6 @@ describe('convertValueToString', () => { it('should escape formula', () => { const result = convertValueToString({ rows: discoverGridContextComplexMock.rows, - rowsFlattened: discoverGridContextComplexMock.rowsFlattened, dataView: discoverGridContextComplexMock.indexPattern, services: discoverServiceMock, columnId: 'array_tags', @@ -478,7 +450,6 @@ describe('convertValueToString', () => { const result2 = convertValueToString({ rows: discoverGridContextComplexMock.rows, - rowsFlattened: discoverGridContextComplexMock.rowsFlattened, dataView: discoverGridContextComplexMock.indexPattern, services: discoverServiceMock, columnId: 'scripted_string', diff --git a/src/plugins/discover/public/utils/convert_value_to_string.ts b/src/plugins/discover/public/utils/convert_value_to_string.ts index 0f00725a69fb2..7c00bc0988cce 100644 --- a/src/plugins/discover/public/utils/convert_value_to_string.ts +++ b/src/plugins/discover/public/utils/convert_value_to_string.ts @@ -9,8 +9,8 @@ import { DataView } from '@kbn/data-views-plugin/public'; import { cellHasFormulas, createEscapeValue } from '@kbn/data-plugin/common'; import { formatFieldValue } from './format_value'; -import { ElasticSearchHit, HitsFlattened } from '../types'; import { DiscoverServices } from '../build_services'; +import { DataTableRecord } from '../types'; interface ConvertedResult { formattedString: string; @@ -20,15 +20,13 @@ interface ConvertedResult { export const convertValueToString = ({ rowIndex, rows, - rowsFlattened, columnId, dataView, services, options, }: { rowIndex: number; - rows: ElasticSearchHit[]; - rowsFlattened: HitsFlattened; + rows: DataTableRecord[]; columnId: string; dataView: DataView; services: DiscoverServices; @@ -37,7 +35,13 @@ export const convertValueToString = ({ }; }): ConvertedResult => { const { fieldFormats } = services; - const rowFlattened = rowsFlattened[rowIndex]; + if (!rows[rowIndex]) { + return { + formattedString: '', + withFormula: false, + }; + } + const rowFlattened = rows[rowIndex].flattened; const value = rowFlattened?.[columnId]; const field = dataView.fields.getByName(columnId); const valuesArray = Array.isArray(value) ? value : [value]; @@ -56,7 +60,7 @@ export const convertValueToString = ({ .map((subValue) => { const formattedValue = formatFieldValue( subValue, - rows[rowIndex], + rows[rowIndex].raw, fieldFormats, dataView, field, diff --git a/src/plugins/discover/public/utils/copy_value_to_clipboard.test.tsx b/src/plugins/discover/public/utils/copy_value_to_clipboard.test.tsx index 3f3beed866eca..842d8147aece1 100644 --- a/src/plugins/discover/public/utils/copy_value_to_clipboard.test.tsx +++ b/src/plugins/discover/public/utils/copy_value_to_clipboard.test.tsx @@ -23,7 +23,6 @@ describe('copyValueToClipboard', () => { const valueToStringConverter: ValueToStringConverter = (rowIndex, columnId, options) => convertValueToString({ rows: discoverGridContextComplexMock.rows, - rowsFlattened: discoverGridContextComplexMock.rowsFlattened, dataView: discoverGridContextComplexMock.indexPattern, services: discoverServiceMock, rowIndex, diff --git a/src/plugins/discover/public/utils/format_hit.test.ts b/src/plugins/discover/public/utils/format_hit.test.ts index 601d88cdedc96..c273a565ff645 100644 --- a/src/plugins/discover/public/utils/format_hit.test.ts +++ b/src/plugins/discover/public/utils/format_hit.test.ts @@ -6,13 +6,15 @@ * Side Public License, v 1. */ -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { indexPatternMock as dataViewMock } from '../__mocks__/index_pattern'; import { formatHit } from './format_hit'; import { discoverServiceMock } from '../__mocks__/services'; +import { DataTableRecord, EsHitRecord } from '../types'; +import { buildDataTableRecord } from './build_data_record'; describe('formatHit', () => { - let hit: estypes.SearchHit; + let row: DataTableRecord; + let hit: EsHitRecord; beforeEach(() => { hit = { _id: '1', @@ -24,6 +26,7 @@ describe('formatHit', () => { bytes: [123], }, }; + row = buildDataTableRecord(hit, dataViewMock); (dataViewMock.getFormatterForField as jest.Mock).mockReturnValue({ convert: (value: unknown) => `formatted:${value}`, }); @@ -35,7 +38,7 @@ describe('formatHit', () => { it('formats a document as expected', () => { const formatted = formatHit( - hit, + row, dataViewMock, ['message', 'extension', 'object.value'], 220, @@ -51,8 +54,16 @@ describe('formatHit', () => { }); it('orders highlighted fields first', () => { + const highlightHit = buildDataTableRecord( + { + ...hit, + highlight: { message: ['%%'] }, + }, + dataViewMock + ); + const formatted = formatHit( - { ...hit, highlight: { message: ['%%'] } }, + highlightHit, dataViewMock, ['message', 'extension', 'object.value'], 220, @@ -69,7 +80,7 @@ describe('formatHit', () => { it('only limits count of pairs based on advanced setting', () => { const formatted = formatHit( - hit, + row, dataViewMock, ['message', 'extension', 'object.value'], 2, @@ -84,7 +95,7 @@ describe('formatHit', () => { it('should not include fields not mentioned in fieldsToShow', () => { const formatted = formatHit( - hit, + row, dataViewMock, ['message', 'object.value'], 220, @@ -100,7 +111,7 @@ describe('formatHit', () => { it('should filter fields based on their real name not displayName', () => { const formatted = formatHit( - hit, + row, dataViewMock, ['bytes'], 220, diff --git a/src/plugins/discover/public/utils/format_hit.ts b/src/plugins/discover/public/utils/format_hit.ts index 6af6a8aa61210..4e3c6160109c6 100644 --- a/src/plugins/discover/public/utils/format_hit.ts +++ b/src/plugins/discover/public/utils/format_hit.ts @@ -9,8 +9,8 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { i18n } from '@kbn/i18n'; import { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; -import { flattenHit } from '@kbn/data-plugin/public'; import { DataView } from '@kbn/data-views-plugin/public'; +import { DataTableRecord } from '../types'; import { formatFieldValue } from './format_value'; const formattedHitCache = new WeakMap(); @@ -26,20 +26,20 @@ type FormattedHit = Array; * @param fieldsToShow A list of fields that should be included in the document summary. */ export function formatHit( - hit: estypes.SearchHit, + hit: DataTableRecord, dataView: DataView, fieldsToShow: string[], maxEntries: number, fieldFormats: FieldFormatsStart ): FormattedHit { - const cached = formattedHitCache.get(hit); + const cached = formattedHitCache.get(hit.raw); if (cached) { return cached; } - const highlights = hit.highlight ?? {}; + const highlights = hit.raw.highlight ?? {}; // Flatten the object using the flattenHit implementation we use across Discover for flattening documents. - const flattened = flattenHit(hit, dataView, { includeIgnoredValues: true, source: true }); + const flattened = hit.flattened; const highlightPairs: Array<[fieldName: string, formattedValue: string]> = []; const sourcePairs: Array<[fieldName: string, formattedValue: string]> = []; @@ -54,7 +54,7 @@ export function formatHit( // Format the raw value using the regular field formatters for that field const formattedValue = formatFieldValue( val, - hit, + hit.raw, fieldFormats, dataView, dataView.fields.getByName(key) @@ -85,6 +85,6 @@ export function formatHit( '', ] as const, ]; - formattedHitCache.set(hit, formatted); + formattedHitCache.set(hit.raw, formatted); return formatted; } diff --git a/src/plugins/discover/public/utils/get_doc_id.tsx b/src/plugins/discover/public/utils/get_doc_id.tsx new file mode 100644 index 0000000000000..5b6b638d3b562 --- /dev/null +++ b/src/plugins/discover/public/utils/get_doc_id.tsx @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { EsHitRecord } from '../types'; +/** + * Returning a generated id of a given ES document, since `_id` can be the same + * when using different indices and shard routing + */ +export const getDocId = (doc: EsHitRecord & { _routing?: string }) => { + const routing = doc._routing ? doc._routing : ''; + return [doc._index, doc._id, routing].join('::'); +};