Merge remote-tracking branch 'upstream/main' into correct-query-schema

common/constants/data_sources.ts

@@ -14,3 +14,8 @@ export const enum QUERY_LANGUAGE {
   SQL = 'SQL',
   DQL = 'DQL',
 }
+export enum DATA_SOURCE_TYPES {
+  DEFAULT_CLUSTER_TYPE = DEFAULT_DATA_SOURCE_TYPE,
+  SPARK = 'spark',
+  S3Glue = 's3glue',
+}

common/constants/shared.ts

@@ -246,3 +246,7 @@ export const VISUALIZATION_ERROR = {
 };
 
 export const S3_DATASOURCE_TYPE = 'S3_DATASOURCE';
+
+export const ASYNC_QUERY_SESSION_ID = 'async-query-session-id';
+
+export const DIRECT_DUMMY_QUERY = 'select 1';

common/types/explorer.ts

@@ -427,3 +427,10 @@ export enum DirectQueryLoadingStatus {
   SCHEDULED = 'SCHEDULED',
   CANCELED = 'CANCELED',
 }
+
+export interface DirectQueryRequest {
+  query: string;
+  lang: string;
+  datasource: string;
+  sessionId?: string;
+}

common/utils/query_session_utils.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { ASYNC_QUERY_SESSION_ID } from '../constants/shared';
+
+export const setAsyncSessionId = (value: string | null) => {
+  if (value !== null) {
+    sessionStorage.setItem(ASYNC_QUERY_SESSION_ID, value);
+  }
+};
+
+export const getAsyncSessionId = () => {
+  return sessionStorage.getItem(ASYNC_QUERY_SESSION_ID);
+};

common/utils/shared.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export function get<T = unknown>(obj: Record<string, any>, path: string, defaultValue?: T): T {
+  return path.split('.').reduce((acc: any, part: string) => acc && acc[part], obj) || defaultValue;
+}

public/components/common/query_utils/index.ts

@@ -245,7 +245,10 @@ export const buildPatternsQuery = (
   return finalQuery;
 };
 
-export const buildQuery = (baseQuery: string, currQuery: string) => baseQuery + '| ' + currQuery;
+export const buildQuery = (baseQuery: string, currQuery: string) => {
+  if (!currQuery) return baseQuery;
+  return `${baseQuery} | ${currQuery}`;
+};
 
 export const buildRawQuery = (query: IQuery, appBaseQuery: string) => {
   if (appBaseQuery && !query.rawQuery.includes(appBaseQuery))

public/components/common/search/sql_search.tsx

@@ -18,14 +18,18 @@ import {
   EuiPopoverFooter,
   EuiToolTip,
 } from '@elastic/eui';
-import { isEqual, lowerCase } from 'lodash';
+import { isEqual } from 'lodash';
 import React, { useEffect, useState } from 'react';
 import { batch, useDispatch, useSelector } from 'react-redux';
+import { QUERY_LANGUAGE } from '../../../../common/constants/data_sources';
 import { APP_ANALYTICS_TAB_ID_REGEX, RAW_QUERY } from '../../../../common/constants/explorer';
-import { DirectQueryLoadingStatus } from '../../../../common/types/explorer';
 import { PPL_NEWLINE_REGEX, PPL_SPAN_REGEX } from '../../../../common/constants/shared';
+import { DirectQueryLoadingStatus, DirectQueryRequest } from '../../../../common/types/explorer';
 import { uiSettingsService } from '../../../../common/utils';
+import { getAsyncSessionId, setAsyncSessionId } from '../../../../common/utils/query_session_utils';
+import { get as getObjValue } from '../../../../common/utils/shared';
 import { useFetchEvents } from '../../../components/event_analytics/hooks';
+import { changeQuery } from '../../../components/event_analytics/redux/slices/query_slice';
 import { usePolling } from '../../../components/hooks/use_polling';
 import { coreRefs } from '../../../framework/core_refs';
 import { SQLService } from '../../../services/requests/sql';
@@ -36,8 +40,6 @@ import {
 } from '../../event_analytics/redux/slices/search_meta_data_slice';
 import { PPLReferenceFlyout } from '../helpers';
 import { Autocomplete } from './autocomplete';
-import { changeQuery } from '../../../components/event_analytics/redux/slices/query_slice';
-import { QUERY_LANGUAGE } from '../../../../common/constants/data_sources';
 export interface IQueryBarProps {
   query: string;
   tempQuery: string;
@@ -101,7 +103,7 @@ export const DirectSearch = (props: any) => {
     stopPolling,
   } = usePolling<any, any>((params) => {
     return sqlService.fetchWithJobId(params);
-  }, 5000);
+  }, 2000);
 
   const requestParams = { tabId };
   const { dispatchOnGettingHis } = useFetchEvents({
@@ -190,14 +192,23 @@ export const DirectSearch = (props: any) => {
       );
     });
     dispatch(updateSearchMetaData({ tabId, data: { isPolling: true, lang } }));
+    const sessionId = getAsyncSessionId();
+    const requestPayload = {
+      lang: lang.toLowerCase(),
+      query: tempQuery || query,
+      datasource: explorerSearchMetadata.datasources[0].label,
+    } as DirectQueryRequest;
+
+    if (sessionId) {
+      requestPayload.sessionId = sessionId;
+    }
+
     sqlService
-      .fetch({
-        lang: lowerCase(lang),
-        query: tempQuery || query,
-        datasource: explorerSearchMetadata.datasources[0].name,
-      })
+      .fetch(requestPayload)
       .then((result) => {
+        setAsyncSessionId(getObjValue(result, 'sessionId', null));
         if (result.queryId) {
+          dispatch(updateSearchMetaData({ tabId, data: { queryId: result.queryId } }));
           startPolling({
             queryId: result.queryId,
           });
@@ -208,8 +219,7 @@ export const DirectSearch = (props: any) => {
       .catch((e) => {
         setIsQueryRunning(false);
         console.error(e);
-      })
-      .finally(() => {});
+      });
   };
 
   useEffect(() => {

public/components/event_analytics/__tests__/__snapshots__/no_results.test.tsx.snap

@@ -106,27 +106,23 @@ exports[`No result component Renders No result component 1`] = `
                     data-test-subj="obsNoResultsTimefilter"
                   >
                     <FormattedMessage
-                      defaultMessage="Expand your time range or modify your query"
+                      defaultMessage="Select a data source, expand your time range, or modify the query"
                       id="observability.noResults.expandYourTimeRangeTitle"
                       values={Object {}}
                     >
                       <span>
-                        Expand your time range or modify your query
+                        Select a data source, expand your time range, or modify the query
                       </span>
                     </FormattedMessage>
                   </h2>
                   <p>
                     <FormattedMessage
-                      defaultMessage="Your query may not match anything in the current time range, 
-                or there may not be any data at all in the currently selected time range. 
-                Try change time range, query filters or choose different time fields."
+                      defaultMessage="After selection, check the time range, query filters, fields, and query"
                       id="observability.noResults.queryMayNotMatchTitle"
                       values={Object {}}
                     >
                       <span>
-                        Your query may not match anything in the current time range, 
-                or there may not be any data at all in the currently selected time range. 
-                Try change time range, query filters or choose different time fields.
+                        After selection, check the time range, query filters, fields, and query
                       </span>
                     </FormattedMessage>
                   </p>

public/components/event_analytics/explorer/datasources/datasources_selection.tsx

@@ -23,7 +23,7 @@ import { reset as resetQueryResults } from '../../redux/slices/query_result_slic
 import { reset as resetVisualization } from '../../redux/slices/visualization_slice';
 import { reset as resetVisConfig } from '../../redux/slices/viualization_config_slice';
 import { reset as resetQuery } from '../../redux/slices/query_slice';
-import { SelectedDataSource } from '../../../../../common/types/explorer';
+import { DirectQueryRequest, SelectedDataSource } from '../../../../../common/types/explorer';
 import { ObservabilityDefaultDataSource } from '../../../../framework/datasources/obs_opensearch_datasource';
 import {
   DATA_SOURCE_TYPE_URL_PARAM_KEY,
@@ -32,7 +32,16 @@ import {
   DEFAULT_DATA_SOURCE_TYPE,
   DEFAULT_DATA_SOURCE_TYPE_NAME,
   DEFAULT_DATA_SOURCE_OBSERVABILITY_DISPLAY_NAME,
+  DATA_SOURCE_TYPES,
+  QUERY_LANGUAGE,
 } from '../../../../../common/constants/data_sources';
+import { SQLService } from '../../../../services/requests/sql';
+import { get as getObjValue } from '../../../../../common/utils/shared';
+import {
+  setAsyncSessionId,
+  getAsyncSessionId,
+} from '../../../../../common/utils/query_session_utils';
+import { DIRECT_DUMMY_QUERY } from '../../../../../common/constants/shared';
 
 const getDataSourceState = (selectedSourceState: SelectedDataSource[]) => {
   if (selectedSourceState.length === 0) return [];
@@ -70,7 +79,8 @@ const removeDataSourceFromURLParams = (currURL: string) => {
 };
 
 export const DataSourceSelection = ({ tabId }: { tabId: string }) => {
-  const { dataSources } = coreRefs;
+  const { dataSources, http } = coreRefs;
+  const sqlService = new SQLService(http!);
   const dispatch = useDispatch();
   const routerContext = useContext(LogExplorerRouterContext);
   const explorerSearchMetadata = useSelector(selectSearchMetaData)[tabId];
@@ -109,6 +119,23 @@ export const DataSourceSelection = ({ tabId }: { tabId: string }) => {
     setSelectedSources(selectedSource);
   };
 
+  const runDummyQuery = (dataSource: string) => {
+    const requestPayload = {
+      lang: QUERY_LANGUAGE.SQL.toLowerCase(),
+      query: DIRECT_DUMMY_QUERY,
+      datasource: dataSource,
+    } as DirectQueryRequest;
+
+    sqlService
+      .fetch(requestPayload)
+      .then((result) => {
+        setAsyncSessionId(getObjValue(result, 'sessionId', null));
+      })
+      .catch((e) => {
+        console.error(e);
+      });
+  };
+
   useEffect(() => {
     setSelectedSources(getDataSourceState(explorerSearchMetadata.datasources));
   }, [explorerSearchMetadata.datasources]);
@@ -166,6 +193,19 @@ export const DataSourceSelection = ({ tabId }: { tabId: string }) => {
     }
   }, []);
 
+  useEffect(() => {
+    // Execute a dummy query to initialize the cluster and obtain a sessionId for subsequent queries.
+    const dsType = explorerSearchMetadata.datasources?.[0]?.type;
+    const dsName = explorerSearchMetadata.datasources?.[0]?.label;
+    if (
+      !getAsyncSessionId() &&
+      [DATA_SOURCE_TYPES.SPARK, DATA_SOURCE_TYPES.S3Glue].includes(dsType) &&
+      dsName
+    ) {
+      runDummyQuery(dsName);
+    }
+  }, [explorerSearchMetadata.datasources]);
+
   /**
    * Process the data source options to display different than discover's group names.
    * Temporary solution for version 2.11.

public/components/event_analytics/explorer/direct_query_running.tsx

@@ -3,18 +3,42 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+import { EuiButton, EuiEmptyPrompt, EuiProgress, EuiSpacer, EuiText } from '@elastic/eui';
 import React from 'react';
 import { useDispatch, useSelector } from 'react-redux';
-import { EuiProgress, EuiEmptyPrompt, EuiButton, EuiText, EuiSpacer } from '@elastic/eui';
+import { DirectQueryLoadingStatus } from '../../../../common/types/explorer';
+import { coreRefs } from '../../../framework/core_refs';
+import { SQLService } from '../../../services/requests/sql';
 import {
   selectSearchMetaData,
   update as updateSearchMetaData,
 } from '../redux/slices/search_meta_data_slice';
-import { DirectQueryLoadingStatus } from '../../../../common/types/explorer';
 
 export const DirectQueryRunning = ({ tabId }: { tabId: string }) => {
   const explorerSearchMeta = useSelector(selectSearchMetaData)[tabId] || {};
   const dispatch = useDispatch();
+  const sqlService = new SQLService(coreRefs.http);
+
+  const cancelQuery = () => {
+    if (explorerSearchMeta.queryId !== '') {
+      sqlService
+        .deleteWithJobId({ queryId: explorerSearchMeta.queryId })
+        .catch((e) => {
+          console.error(e);
+        })
+        .finally(() => {
+          dispatch(
+            updateSearchMetaData({
+              tabId,
+              data: {
+                isPolling: false,
+              },
+            })
+          );
+        });
+    }
+  };
+
   return (
     <EuiEmptyPrompt
       icon={<EuiProgress size="xs" color="accent" />}
@@ -25,19 +49,7 @@ export const DirectQueryRunning = ({ tabId }: { tabId: string }) => {
             Status: {explorerSearchMeta.status ?? DirectQueryLoadingStatus.SCHEDULED}
           </EuiText>
           <EuiSpacer size="s" />
-          <EuiButton
-            color="success"
-            onClick={() => {
-              dispatch(
-                updateSearchMetaData({
-                  tabId,
-                  data: {
-                    isPolling: false,
-                  },
-                })
-              );
-            }}
-          >
+          <EuiButton color="success" onClick={cancelQuery}>
             Cancel
           </EuiButton>
         </>

public/components/event_analytics/explorer/explorer.tsx

@@ -258,6 +258,12 @@ export const Explorer = ({
       ...TIME_INTERVAL_OPTIONS,
     ]);
     selectedIntervalRef.current = { text: 'Auto', value: 'auto_' + minInterval };
+    dispatch(
+      updateCountDistribution({
+        tabId,
+        data: { selectedInterval: selectedIntervalRef.current.value.replace(/^auto_/, '') },
+      })
+    );
   };
 
   useEffect(() => {
@@ -495,13 +501,18 @@ export const Explorer = ({
                           selectedIntervalRef.current = timeIntervalOptions[intervalOptionsIndex];
                           getPatterns(intrv, getErrorHandler('Error fetching patterns'));
                         }}
-                        stateInterval={selectedIntervalRef.current?.value}
+                        stateInterval={
+                          countDistribution.selectedInterval || selectedIntervalRef.current?.value
+                        }
                         startTime={appLogEvents ? startTime : dateRange[0]}
                         endTime={appLogEvents ? endTime : dateRange[1]}
                       />
+                      <EuiSpacer size="s" />
                       <CountDistribution
                         countDistribution={countDistribution}
-                        selectedInterval={selectedIntervalRef.current?.value}
+                        selectedInterval={
+                          countDistribution.selectedInterval || selectedIntervalRef.current?.value
+                        }
                         startTime={appLogEvents ? startTime : dateRange[0]}
                         endTime={appLogEvents ? endTime : dateRange[1]}
                       />
@@ -783,6 +794,8 @@ export const Explorer = ({
     subType,
     selectedCustomPanelOptions,
     explorerSearchMeta,
+    selectedIntervalRef.current,
+    countDistribution,
   ]);
 
   const liveTailLoop = async (
@@ -794,7 +807,7 @@ export const Explorer = ({
     setLiveTailName(name);
     setLiveTailTabId((curSelectedTabId.current as unknown) as string);
     setIsLiveTailOn(true);
-    setToast('Live tail On', 'success');
+    setToast('Live tail On', 'success', '', 'right', 2000);
     setIsLiveTailPopoverOpen(false);
     setLiveTimestamp(
       dateMath.parse(endingTime, { roundUp: true })?.utc().format(DATE_PICKER_FORMAT) || ''
@@ -819,7 +832,7 @@ export const Explorer = ({
     setIsLiveTailOn(false);
     setLiveHits(0);
     setIsLiveTailPopoverOpen(false);
-    if (isLiveTailOnRef.current) setToast('Live tail Off', 'danger');
+    if (isLiveTailOnRef.current) setToast('Live tail Off', 'danger', '', 'right', 2000);
   };
 
   useEffect(() => {