Populated source index with data streams in create rollup/transform f…

…lows. Signed-off-by: Ketan Verma <ketan9495@gmail.com>
opensearch-project · Jun 17, 2021 · 8799f30 · 8799f30
1 parent e0dadb8
commit 8799f30
Show file tree

Hide file tree

Showing 6 changed files with 66 additions and 10 deletions.
diff --git a/public/pages/CreateRollup/components/RollupIndices/RollupIndices.tsx b/public/pages/CreateRollup/components/RollupIndices/RollupIndices.tsx
@@ -72,13 +72,22 @@ export default class RollupIndices extends Component<RollupIndicesProps, RollupI
     this.setState({ isLoading: true, indexOptions: [] });
     try {
       const queryObject = { from: 0, size: 10, search: searchValue, sortDirection: "desc", sortField: "index", showDataStreams: true };
-      const getIndicesResponse = await indexService.getIndices(queryObject);
+      const [getIndicesResponse, getDataStreamsResponse] = await Promise.all([
+        indexService.getIndices(queryObject),
+        indexService.getDataStreams({ search: searchValue }),
+      ]);
+
       if (getIndicesResponse.ok) {
         const options = searchValue.trim() ? [{ label: `${searchValue}*` }] : [];
+        const dataStreams = getDataStreamsResponse.ok
+          ? getDataStreamsResponse.response.dataStreams.map((ds) => ({
+              label: ds.name,
+            }))
+          : [];
         const indices = getIndicesResponse.response.indices.map((index: IndexItem) => ({
           label: index.index,
         }));
-        this.setState({ indexOptions: options.concat(indices), targetIndexOptions: indices });
+        this.setState({ indexOptions: options.concat(dataStreams, indices), targetIndexOptions: indices });
       } else {
         if (getIndicesResponse.error.startsWith("[index_not_found_exception]")) {
           this.context.notifications.toasts.addDanger("No index available");

diff --git a/public/pages/CreateRollup/containers/CreateRollupForm/CreateRollupForm.test.tsx b/public/pages/CreateRollup/containers/CreateRollupForm/CreateRollupForm.test.tsx
@@ -36,6 +36,7 @@ import { ModalProvider, ModalRoot } from "../../../../components/Modal";
 import { BREADCRUMBS, ROUTES } from "../../../../utils/constants";
 import CreateRollupForm from "./CreateRollupForm";
 import { CoreServicesContext } from "../../../../components/core_services";
+import { DataStream } from "../../../../../server/models/interfaces";
 
 const indices = [
   {
@@ -52,6 +53,24 @@ const indices = [
   },
 ];
 
+const dataStreams: DataStream[] = [
+  {
+    name: "data_stream_1",
+    indices: [
+      {
+        index_uuid: "uuid",
+        index_name: "name",
+      },
+    ],
+    status: "GREEN",
+    timestamp_field: {
+      name: "@timestamp",
+    },
+    template: "template",
+    generation: 1,
+  },
+];
+
 const sampleMapping = {
   index_1: {
     mappings: {
@@ -230,6 +249,11 @@ describe("<CreateRollupForm /> creation", () => {
     response: { indices, totalIndices: 1 },
   });
 
+  browserServicesMock.indexService.getDataStreams = jest.fn().mockResolvedValue({
+    ok: true,
+    response: { dataStreams, totalDataStreams: 1 },
+  });
+
   browserServicesMock.rollupService.getMappings = jest.fn().mockResolvedValue({
     ok: true,
     response: sampleMapping,

diff --git a/public/pages/CreateTransform/components/TransformIndices/TransformIndices.tsx b/public/pages/CreateTransform/components/TransformIndices/TransformIndices.tsx
@@ -82,13 +82,22 @@ export default class TransformIndices extends Component<TransformIndicesProps, T
     this.setState({ isLoading: true, indexOptions: [] });
     try {
       const queryObject = { from: 0, size: 10, search: searchValue, sortDirection: "desc", sortField: "index", showDataStreams: true };
-      const getIndicesResponse = await indexService.getIndices(queryObject);
+      const [getIndicesResponse, getDataStreamsResponse] = await Promise.all([
+        indexService.getIndices(queryObject),
+        indexService.getDataStreams({ search: searchValue }),
+      ]);
+
       if (getIndicesResponse.ok) {
         const options = searchValue.trim() ? [{ label: `${searchValue}*` }] : [];
+        const dataStreams = getDataStreamsResponse.ok
+          ? getDataStreamsResponse.response.dataStreams.map((ds) => ({
+              label: ds.name,
+            }))
+          : [];
         const indices = getIndicesResponse.response.indices.map((index: IndexItem) => ({
           label: index.index,
         }));
-        this.setState({ indexOptions: options.concat(indices), targetIndexOptions: indices });
+        this.setState({ indexOptions: options.concat(dataStreams, indices), targetIndexOptions: indices });
       } else {
         if (getIndicesResponse.error.startsWith("[index_not_found_exception]")) {
           this.context.notifications.toasts.addDanger("No index available");

diff --git a/public/services/IndexService.ts b/public/services/IndexService.ts
@@ -48,9 +48,9 @@ export default class IndexService {
     return response;
   };
 
-  getDataStreams = async (): Promise<ServerResponse<GetDataStreamsResponse>> => {
+  getDataStreams = async (queryObject: HttpFetchQuery): Promise<ServerResponse<GetDataStreamsResponse>> => {
     const url = `..${NODE_API._DATA_STREAMS}`;
-    return await this.httpClient.get(url);
+    return await this.httpClient.get(url, { query: queryObject });
   };
 
   applyPolicy = async (indices: string[], policyId: string): Promise<ServerResponse<ApplyPolicyResponse>> => {

diff --git a/server/routes/dataStreams.ts b/server/routes/dataStreams.ts
@@ -9,6 +9,7 @@
  * GitHub history for details.
  */
 
+import { schema } from "@osd/config-schema";
 import { NodeServices } from "../models/interfaces";
 import { NODE_API } from "../../utils/constants";
 import { IRouter } from "../../../../src/core/server";
@@ -19,7 +20,11 @@ export default function (services: NodeServices, router: IRouter) {
   router.get(
     {
       path: NODE_API._DATA_STREAMS,
-      validate: {},
+      validate: {
+        query: schema.object({
+          search: schema.maybe(schema.string()),
+        }),
+      },
     },
     dataStreamService.getDataStreams
   );

diff --git a/server/services/DataStreamService.ts b/server/services/DataStreamService.ts
@@ -33,8 +33,12 @@ export default class DataStreamService {
     response: OpenSearchDashboardsResponseFactory
   ): Promise<IOpenSearchDashboardsResponse<ServerResponse<GetDataStreamsResponse>>> => {
     try {
+      const { search } = request.query as {
+        search?: string;
+      };
+
       const client = this.osDriver.asScoped(request);
-      const dataStreams = await getDataStreams(client);
+      const dataStreams = await getDataStreams(client, search);
 
       return response.custom({
         statusCode: 200,
@@ -59,9 +63,14 @@ export default class DataStreamService {
   };
 }
 
-export async function getDataStreams({ callAsCurrentUser: callWithRequest }: ILegacyScopedClusterClient): Promise<DataStream[]> {
+export async function getDataStreams(
+  { callAsCurrentUser: callWithRequest }: ILegacyScopedClusterClient,
+  search?: string
+): Promise<DataStream[]> {
+  const searchPattern = search ? `*${search}*` : "*";
+
   const dataStreamsResponse = await callWithRequest("transport.request", {
-    path: "/_data_stream",
+    path: `/_data_stream/${searchPattern}`,
     method: "GET",
   });