[ML] Switch from normal sampling to random sampler for Index data visualizer table

src/plugins/discover/public/application/main/components/field_stats_table/field_stats_table.tsx

@@ -24,6 +24,24 @@ import { useDiscoverServices } from '../../../../hooks/use_discover_services';
 import { FIELD_STATISTICS_LOADED } from './constants';
 import type { GetStateReturn } from '../../services/discover_state';
 import { AvailableFields$, DataRefetch$, DataTotalHits$ } from '../../hooks/use_saved_search';
+export interface RandomSamplingOption {
+  mode: 'random_sampling';
+  seed: string;
+  probability: number;
+}
+
+export interface NormalSamplingOption {
+  mode: 'normal_sampling';
+  seed: string;
+  shardSize: number;
+}
+
+export interface NoSamplingOption {
+  mode: 'no_sampling';
+  seed: string;
+}
+
+export type SamplingOption = RandomSamplingOption | NormalSamplingOption | NoSamplingOption;
 
 export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput {
   dataView: DataView;
@@ -39,6 +57,7 @@ export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput {
   sessionId?: string;
   fieldsToFetch?: string[];
   totalDocuments?: number;
+  samplingOption?: SamplingOption;
 }
 export interface DataVisualizerGridEmbeddableOutput extends EmbeddableOutput {
   showDistributions?: boolean;
@@ -163,6 +182,11 @@ export const FieldStatisticsTable = (props: FieldStatisticsTableProps) => {
         totalDocuments: savedSearchDataTotalHits$
           ? savedSearchDataTotalHits$.getValue()?.result
           : undefined,
+        samplingOption: {
+          mode: 'normal_sampling',
+          shardSize: 5000,
+          seed: searchSessionId,
+        } as NormalSamplingOption,
       });
       embeddable.reload();
     }

x-pack/packages/ml/is_populated_object/src/is_populated_object.ts

@@ -22,10 +22,10 @@
  *      Otherwise you'd just satisfy TS requirements but might still
  *      run into runtime issues.
  */
-export const isPopulatedObject = <U extends string = string>(
+export const isPopulatedObject = <U extends string = string, T extends unknown = unknown>(
   arg: unknown,
   requiredAttributes: U[] = []
-): arg is Record<U, unknown> => {
+): arg is Record<U, T> => {
   return (
     typeof arg === 'object' &&
     arg !== null &&

x-pack/plugins/data_visualizer/common/types/field_request_config.ts

@@ -64,9 +64,7 @@ export interface FieldVisStats {
   max?: number;
   median?: number;
   min?: number;
-  topValues?: Array<{ key: number | string; doc_count: number }>;
-  topValuesSampleSize?: number;
-  topValuesSamplerShardSize?: number;
+  topValues?: Array<{ key: number | string; doc_count: number; percent: number }>;
   examples?: Array<string | GeoPointExample | object>;
   timeRangeEarliest?: number;
   timeRangeLatest?: number;

x-pack/plugins/data_visualizer/common/types/field_stats.ts

@@ -11,6 +11,25 @@ import { IKibanaSearchResponse } from '@kbn/data-plugin/common';
 import { isPopulatedObject } from '@kbn/ml-is-populated-object';
 import { TimeBucketsInterval } from '../services/time_buckets';
 
+export interface RandomSamplingOption {
+  mode: 'random_sampling';
+  seed: string;
+  probability: number;
+}
+
+export interface NormalSamplingOption {
+  mode: 'normal_sampling';
+  seed: string;
+  shardSize: number;
+}
+
+export interface NoSamplingOption {
+  mode: 'no_sampling';
+  seed: string;
+}
+
+export type SamplingOption = RandomSamplingOption | NormalSamplingOption | NoSamplingOption;
+
 export interface FieldData {
   fieldName: string;
   existsInDocs: boolean;
@@ -54,7 +73,7 @@ export const isIKibanaSearchResponse = (arg: unknown): arg is IKibanaSearchRespo
 
 export interface NumericFieldStats {
   fieldName: string;
-  count: number;
+  count?: number;
   min: number;
   max: number;
   avg: number;
@@ -86,7 +105,8 @@ export interface BooleanFieldStats {
   count: number;
   trueCount: number;
   falseCount: number;
-  [key: string]: number | string;
+  topValues: Bucket[];
+  topValuesSampleSize: number;
 }
 
 export interface DocumentCountStats {
@@ -186,6 +206,9 @@ export interface FieldStatsCommonRequestParams {
   intervalMs?: number;
   query: estypes.QueryDslQueryContainer;
   maxExamples?: number;
+  samplingProbability: number | null;
+  browserSessionSeed: number;
+  samplingOption: SamplingOption;
 }
 
 export interface OverallStatsSearchStrategyParams {
@@ -202,6 +225,8 @@ export interface OverallStatsSearchStrategyParams {
   aggregatableFields: string[];
   nonAggregatableFields: string[];
   fieldsToFetch?: string[];
+  browserSessionSeed: number;
+  samplingOption: SamplingOption;
 }
 
 export interface FieldStatsSearchStrategyReturnBase {
@@ -238,3 +263,20 @@ export interface Field {
 export interface Aggs {
   [key: string]: estypes.AggregationsAggregationContainer;
 }
+
+export const EMBEDDABLE_SAMPLER_OPTION = {
+  RANDOM: 'random_sampling',
+  NORMAL: 'normal_sampling',
+};
+export type FieldStatsEmbeddableSamplerOption =
+  typeof EMBEDDABLE_SAMPLER_OPTION[keyof typeof EMBEDDABLE_SAMPLER_OPTION];
+
+export function isRandomSamplingOption(arg: SamplingOption): arg is RandomSamplingOption {
+  return arg.mode === 'random_sampling';
+}
+export function isNormalSamplingOption(arg: SamplingOption): arg is NormalSamplingOption {
+  return arg.mode === 'normal_sampling';
+}
+export function isNoSamplingOption(arg: SamplingOption): arg is NoSamplingOption {
+  return arg.mode === 'no_sampling' || (arg.mode === 'random_sampling' && arg.probability === 1);
+}

x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/document_count_content.tsx

@@ -20,7 +20,7 @@ import {
   EuiFormRow,
 } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
-import { sortedIndex } from 'lodash';
+import { debounce, sortedIndex } from 'lodash';
 import { FormattedMessage } from '@kbn/i18n-react';
 import { isDefined } from '../../util/is_defined';
 import type { DocumentCountChartPoint } from './document_count_chart';
@@ -64,6 +64,16 @@ export const DocumentCountContent: FC<Props> = ({
     setShowSamplingOptionsPopover(false);
   }, [setShowSamplingOptionsPopover]);
 
+  // eslint-disable-next-line react-hooks/exhaustive-deps
+  const updateSamplingProbability = useCallback(
+    debounce((p) => {
+      if (setSamplingProbability) {
+        setSamplingProbability(p);
+      }
+    }, 100),
+    [setSamplingProbability]
+  );
+
   const calloutInfoMessage = useMemo(() => {
     switch (randomSamplerPreference) {
       case RANDOM_SAMPLER_OPTION.OFF:
@@ -124,7 +134,6 @@ export const DocumentCountContent: FC<Props> = ({
   return (
     <>
       <EuiFlexGroup alignItems="center" gutterSize="xs">
-        <TotalCountHeader totalCount={totalCount} approximate={approximate} loading={loading} />
         <EuiFlexItem grow={false}>
           <EuiPopover
             data-test-subj="dvRandomSamplerOptionsPopover"
@@ -210,9 +219,7 @@ export const DocumentCountContent: FC<Props> = ({
                             ? closestPrev
                             : closestNext;
 
-                        if (setSamplingProbability) {
-                          setSamplingProbability(closestProbability / 100);
-                        }
+                        updateSamplingProbability(closestProbability / 100);
                       }}
                       step={RANDOM_SAMPLER_STEP}
                       data-test-subj="dvRandomSamplerProbabilityRange"
@@ -226,6 +233,7 @@ export const DocumentCountContent: FC<Props> = ({
           </EuiPopover>
           <EuiFlexItem />
         </EuiFlexItem>
+        <TotalCountHeader totalCount={totalCount} approximate={approximate} loading={loading} />
       </EuiFlexGroup>
       <DocumentCountChart
         chartPoints={chartPoints}

x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/fields_stats_grid.tsx

@@ -112,6 +112,7 @@ export const FieldsStatsGrid: FC<Props> = ({ results }) => {
         pageState={dataVisualizerListState}
         updatePageState={setDataVisualizerListState}
         getItemIdToExpandedRowMap={getItemIdToExpandedRowMap}
+        overallStatsRunning={false}
       />
     </div>
   );

x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/boolean_content.tsx

@@ -10,7 +10,6 @@ import { EuiSpacer } from '@elastic/eui';
 import { Axis, BarSeries, Chart, Settings, ScaleType } from '@elastic/charts';
 
 import { FormattedMessage } from '@kbn/i18n-react';
-import { i18n } from '@kbn/i18n';
 import { TopValues } from '../../../top_values';
 import type { FieldDataRowProps } from '../../types/field_data_row';
 import { ExpandedRowFieldHeader } from '../expanded_row_field_header';
@@ -45,32 +44,13 @@ export const BooleanContent: FC<FieldDataRowProps> = ({ config, onAddFilter }) =
   const theme = useDataVizChartTheme();
   if (!formattedPercentages) return null;
 
-  const { trueCount, falseCount, count } = formattedPercentages;
-  const stats = {
-    ...config.stats,
-    topValues: [
-      {
-        key: i18n.translate(
-          'xpack.dataVisualizer.dataGrid.fieldExpandedRow.booleanContent.trueCountLabel',
-          { defaultMessage: 'true' }
-        ),
-        doc_count: trueCount ?? 0,
-      },
-      {
-        key: i18n.translate(
-          'xpack.dataVisualizer.dataGrid.fieldExpandedRow.booleanContent.falseCountLabel',
-          { defaultMessage: 'false' }
-        ),
-        doc_count: falseCount ?? 0,
-      },
-    ],
-  };
+  const { count } = formattedPercentages;
   return (
     <ExpandedRowContent dataTestSubj={'dataVisualizerBooleanContent'}>
       <DocumentStatsTable config={config} />
 
       <TopValues
-        stats={stats}
+        stats={config.stats}
         fieldFormat={fieldFormat}
         barColor="success"
         onAddFilter={onAddFilter}

x-pack/plugins/data_visualizer/public/application/common/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx

@@ -6,7 +6,7 @@
  */
 
 import React, { FC, useMemo } from 'react';
-import { EuiSpacer, EuiText, htmlIdGenerator } from '@elastic/eui';
+import { EuiText, htmlIdGenerator } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n-react';
 import {
@@ -18,6 +18,8 @@ import {
   VectorLayerDescriptor,
 } from '@kbn/maps-plugin/common';
 import { EMSTermJoinConfig } from '@kbn/maps-plugin/public';
+import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/field-types';
+import { useDataVisualizerKibana } from '../../../../../kibana_context';
 import { EmbeddedMapComponent } from '../../../embedded_map';
 import { FieldVisStats } from '../../../../../../../common/types';
 import { ExpandedRowPanel } from './expanded_row_panel';
@@ -97,13 +99,55 @@ interface Props {
 }
 
 export const ChoroplethMap: FC<Props> = ({ stats, suggestion }) => {
-  const { fieldName, isTopValuesSampled, topValues, topValuesSamplerShardSize } = stats!;
+  const {
+    services: {
+      data: { fieldFormats },
+    },
+  } = useDataVisualizerKibana();
+
+  const { fieldName, isTopValuesSampled, topValues, sampleCount, totalDocuments } = stats!;
 
   const layerList: VectorLayerDescriptor[] = useMemo(
     () => [getChoroplethTopValuesLayer(fieldName || '', topValues || [], suggestion)],
     [suggestion, fieldName, topValues]
   );
 
+  const countsElement = totalDocuments ? (
+    <EuiText color="subdued" size="xs">
+      {isTopValuesSampled ? (
+        <FormattedMessage
+          id="xpack.dataVisualizer.dataGrid.fieldExpandedRow.choroplethMapTopValues.calculatedFromSampleRecordsLabel"
+          defaultMessage="Calculated from {sampledDocumentsFormatted} sample {sampledDocuments, plural, one {record} other {records}}."
+          values={{
+            sampledDocuments: sampleCount,
+            sampledDocumentsFormatted: (
+              <strong>
+                {fieldFormats
+                  .getDefaultInstance(KBN_FIELD_TYPES.NUMBER, [ES_FIELD_TYPES.INTEGER])
+                  .convert(sampleCount)}
+              </strong>
+            ),
+          }}
+        />
+      ) : (
+        <FormattedMessage
+          id="xpack.dataVisualizer.dataGrid.fieldExpandedRow.choroplethMapTopValues.calculatedFromTotalRecordsLabel"
+          defaultMessage="Calculated from {totalDocumentsFormatted} {totalDocuments, plural, one {record} other {records}}."
+          values={{
+            totalDocuments,
+            totalDocumentsFormatted: (
+              <strong>
+                {fieldFormats
+                  .getDefaultInstance(KBN_FIELD_TYPES.NUMBER, [ES_FIELD_TYPES.INTEGER])
+                  .convert(totalDocuments ?? 0)}
+              </strong>
+            ),
+          }}
+        />
+      )}
+    </EuiText>
+  ) : null;
+
   return (
     <ExpandedRowPanel
       dataTestSubj={'fileDataVisualizerChoroplethMapTopValues'}
@@ -114,20 +158,7 @@ export const ChoroplethMap: FC<Props> = ({ stats, suggestion }) => {
         <EmbeddedMapComponent layerList={layerList} />
       </div>
 
-      {isTopValuesSampled === true && (
-        <div>
-          <EuiSpacer size={'s'} />
-          <EuiText size="xs" textAlign={'center'}>
-            <FormattedMessage
-              id="xpack.dataVisualizer.dataGrid.fieldExpandedRow.choroplethMapTopValues.calculatedFromSampleDescription"
-              defaultMessage="Calculated from sample of {topValuesSamplerShardSize} documents per shard"
-              values={{
-                topValuesSamplerShardSize,
-              }}
-            />
-          </EuiText>
-        </div>
-      )}
+      {countsElement}
     </ExpandedRowPanel>
   );
 };