From 3005dc0fa957a9bbd0fe44a06da0f56f01ca6626 Mon Sep 17 00:00:00 2001 From: Annie Date: Tue, 22 Dec 2020 14:33:11 -0800 Subject: [PATCH 01/15] Changes to getRollups, but with error of missing rollupID params --- public/services/RollupService.ts | 6 ++-- server/clusters/ism/ismPlugin.ts | 28 ++++++++++++++++++ server/models/interfaces.ts | 2 +- server/routes/rollups.ts | 15 +++++----- server/services/RollupService.ts | 51 +++++++++++--------------------- 5 files changed, 59 insertions(+), 43 deletions(-) diff --git a/public/services/RollupService.ts b/public/services/RollupService.ts index e6395ba7..02f8a94f 100644 --- a/public/services/RollupService.ts +++ b/public/services/RollupService.ts @@ -18,6 +18,7 @@ import { PutRollupResponse, GetRollupsResponse, GetFieldsResponse } from "../../ import { ServerResponse } from "../../server/models/types"; import { NODE_API } from "../../utils/constants"; import { DocumentRollup, Rollup } from "../../models/interfaces"; +import { RollupQueryParams } from "../pages/Rollups/models/interfaces"; export default class RollupService { httpClient: HttpSetup; @@ -26,9 +27,10 @@ export default class RollupService { this.httpClient = httpClient; } - getRollups = async (queryObject: object): Promise> => { + getRollups = async (queryObject: RollupQueryParams): Promise> => { let url = `..${NODE_API.ROLLUPS}`; - const response = (await this.httpClient.get(url, { query: queryObject })) as ServerResponse; + // const response = (await this.httpClient.get(url, {query :{ rollupID: "test",from: 2, size:20, search: null, sortField: "name", sortDirection: "desc"}})) as ServerResponse; + const response = (await this.httpClient.get(url)) as ServerResponse; return response; }; diff --git a/server/clusters/ism/ismPlugin.ts b/server/clusters/ism/ismPlugin.ts index 464d47aa..2d942308 100644 --- a/server/clusters/ism/ismPlugin.ts +++ b/server/clusters/ism/ismPlugin.ts @@ -171,6 +171,34 @@ export default function ismPlugin(Client: any, config: any, components: any) { method: "GET", }); + ism.getRollups = ca({ + url: { + fmt: `${API.ROLLUP_JOBS_BASE}`, + req: { + from: { + type: "number", + required: true, + }, + size: { + type: "number", + required: true, + }, + search: { + type: "string", + }, + sortField: { + type: "string", + required: true, + }, + sortDirection: { + type: "string", + required: true, + }, + }, + }, + method: "GET", + }); + ism.createRollup = ca({ url: { fmt: `${API.ROLLUP_JOBS_BASE}/<%=rollupId%>?refresh=wait_for`, diff --git a/server/models/interfaces.ts b/server/models/interfaces.ts index 857dfa04..3a4c9a35 100644 --- a/server/models/interfaces.ts +++ b/server/models/interfaces.ts @@ -72,7 +72,7 @@ export interface GetFieldsResponse { export interface GetRollupsResponse { rollups: DocumentRollup[]; - totalRollups: number; + totalRollups: number | undefined; metadata: any; } diff --git a/server/routes/rollups.ts b/server/routes/rollups.ts index 9e32e397..2386fae9 100644 --- a/server/routes/rollups.ts +++ b/server/routes/rollups.ts @@ -25,13 +25,14 @@ export default function (services: NodeServices, router: IRouter) { { path: NODE_API.ROLLUPS, validate: { - query: schema.object({ - from: schema.number(), - size: schema.number(), - search: schema.string(), - sortField: schema.string(), - sortDirection: schema.string(), - }), + // query: schema.object({ + // from: schema.number(), + // size: schema.number(), + // search: schema.string(), + // sortField: schema.string(), + // sortDirection: schema.string(), + // }), + query: schema.any(), }, }, rollupService.getRollups diff --git a/server/services/RollupService.ts b/server/services/RollupService.ts index b58859d4..3cd2b0b3 100644 --- a/server/services/RollupService.ts +++ b/server/services/RollupService.ts @@ -193,7 +193,7 @@ export default class RollupService { const { id } = request.params as { id: string }; const params = { rollupId: id }; const { callAsCurrentUser: callWithRequest } = this.esDriver.asScoped(request); - const getResponse = await callWithRequest("ism.getRollup", params); + const getResponse = await callWithRequest("ism.getRollups", params); const metadata = await callWithRequest("ism.explainRollup", params); const rollup = _.get(getResponse, "rollup", null); const seqNo = _.get(getResponse, "_seq_no"); @@ -312,55 +312,40 @@ export default class RollupService { } }; - /** - * Performs a fuzzy search request on rollup id - */ getRollups = async ( context: RequestHandlerContext, request: KibanaRequest, response: KibanaResponseFactory ): Promise>> => { try { - const { from, size, search, sortDirection, sortField } = request.query as { - from: string; - size: string; + const { from, size, search, sortDirection, sortField } = request.params as { + from: number; + size: number; search: string; sortDirection: string; sortField: string; }; - const rollupSorts: RollupsSort = { - id: "rollup.rollup_id.keyword", - "rollup.rollup.description": "rollup.description.keyword", - "rollup.rollup.last_updated_time": "rollup.last_updated_time", - }; const params = { - index: INDEX.OPENDISTRO_ISM_CONFIG, - seq_no_primary_term: true, - body: { - size, - from, - sort: rollupSorts[sortField] ? [{ [rollupSorts[sortField]]: sortDirection }] : [], - query: { - bool: { - filter: [{ exists: { field: "rollup" } }], - must: getMustQuery("rollup.rollup_id", search), - }, - }, - }, + rollupID: "test", + from, + size, + search, + sortField, + sortDirection, }; const { callAsCurrentUser: callWithRequest } = this.esDriver.asScoped(request); - const searchResponse: SearchResponse = await callWithRequest("search", params); - const totalRollups = searchResponse.hits.total.value; - - const rollups = searchResponse.hits.hits.map((hit) => ({ - _seqNo: hit._seq_no as number, - _primaryTerm: hit._primary_term as number, - _id: hit._id, - rollup: hit._source, + const getRollupResponse: GetRollupsResponse = await callWithRequest("ism.getRollup", params); + const totalRollups = getRollupResponse.totalRollups; + const rollups = getRollupResponse.rollups.map((rollup) => ({ + _seqNo: rollup._seqNo as number, + _primaryTerm: rollup._primaryTerm as number, + _id: rollup._id, + rollup: rollup, metadata: null, })); + if (totalRollups) { const ids = rollups.map((rollup) => rollup._id).join(","); const explainResponse = await this.explainRollup(context, request, response, ids); From c13d2b2f03fd656d7a2dd28685c9c9213181903e Mon Sep 17 00:00:00 2001 From: Annie Date: Tue, 22 Dec 2020 16:06:59 -0800 Subject: [PATCH 02/15] Modify callWithRequeset to use ism.getRollups --- public/services/RollupService.ts | 3 +-- server/routes/rollups.ts | 7 ------- server/services/RollupService.ts | 5 ++--- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/public/services/RollupService.ts b/public/services/RollupService.ts index 02f8a94f..77dc9c97 100644 --- a/public/services/RollupService.ts +++ b/public/services/RollupService.ts @@ -29,8 +29,7 @@ export default class RollupService { getRollups = async (queryObject: RollupQueryParams): Promise> => { let url = `..${NODE_API.ROLLUPS}`; - // const response = (await this.httpClient.get(url, {query :{ rollupID: "test",from: 2, size:20, search: null, sortField: "name", sortDirection: "desc"}})) as ServerResponse; - const response = (await this.httpClient.get(url)) as ServerResponse; + const response = (await this.httpClient.get(url, { query: queryObject })) as ServerResponse; return response; }; diff --git a/server/routes/rollups.ts b/server/routes/rollups.ts index 2386fae9..027e4225 100644 --- a/server/routes/rollups.ts +++ b/server/routes/rollups.ts @@ -25,13 +25,6 @@ export default function (services: NodeServices, router: IRouter) { { path: NODE_API.ROLLUPS, validate: { - // query: schema.object({ - // from: schema.number(), - // size: schema.number(), - // search: schema.string(), - // sortField: schema.string(), - // sortDirection: schema.string(), - // }), query: schema.any(), }, }, diff --git a/server/services/RollupService.ts b/server/services/RollupService.ts index 3cd2b0b3..6d595858 100644 --- a/server/services/RollupService.ts +++ b/server/services/RollupService.ts @@ -318,7 +318,7 @@ export default class RollupService { response: KibanaResponseFactory ): Promise>> => { try { - const { from, size, search, sortDirection, sortField } = request.params as { + const { from, size, search, sortDirection, sortField } = request.query as { from: number; size: number; search: string; @@ -327,7 +327,6 @@ export default class RollupService { }; const params = { - rollupID: "test", from, size, search, @@ -336,7 +335,7 @@ export default class RollupService { }; const { callAsCurrentUser: callWithRequest } = this.esDriver.asScoped(request); - const getRollupResponse: GetRollupsResponse = await callWithRequest("ism.getRollup", params); + const getRollupResponse: GetRollupsResponse = await callWithRequest("ism.getRollups", params); const totalRollups = getRollupResponse.totalRollups; const rollups = getRollupResponse.rollups.map((rollup) => ({ _seqNo: rollup._seqNo as number, From 0d2e800e40246c23533bbd9bc739d601e992a40b Mon Sep 17 00:00:00 2001 From: Annie Date: Wed, 23 Dec 2020 14:50:03 -0800 Subject: [PATCH 03/15] Clean up code --- public/services/RollupService.ts | 3 +-- server/routes/rollups.ts | 8 +++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/public/services/RollupService.ts b/public/services/RollupService.ts index 77dc9c97..e6395ba7 100644 --- a/public/services/RollupService.ts +++ b/public/services/RollupService.ts @@ -18,7 +18,6 @@ import { PutRollupResponse, GetRollupsResponse, GetFieldsResponse } from "../../ import { ServerResponse } from "../../server/models/types"; import { NODE_API } from "../../utils/constants"; import { DocumentRollup, Rollup } from "../../models/interfaces"; -import { RollupQueryParams } from "../pages/Rollups/models/interfaces"; export default class RollupService { httpClient: HttpSetup; @@ -27,7 +26,7 @@ export default class RollupService { this.httpClient = httpClient; } - getRollups = async (queryObject: RollupQueryParams): Promise> => { + getRollups = async (queryObject: object): Promise> => { let url = `..${NODE_API.ROLLUPS}`; const response = (await this.httpClient.get(url, { query: queryObject })) as ServerResponse; return response; diff --git a/server/routes/rollups.ts b/server/routes/rollups.ts index 027e4225..9e32e397 100644 --- a/server/routes/rollups.ts +++ b/server/routes/rollups.ts @@ -25,7 +25,13 @@ export default function (services: NodeServices, router: IRouter) { { path: NODE_API.ROLLUPS, validate: { - query: schema.any(), + query: schema.object({ + from: schema.number(), + size: schema.number(), + search: schema.string(), + sortField: schema.string(), + sortDirection: schema.string(), + }), }, }, rollupService.getRollups From dd52563c934ea4080cf17c6134bd9b5de892a053 Mon Sep 17 00:00:00 2001 From: Annie Date: Wed, 23 Dec 2020 16:18:54 -0800 Subject: [PATCH 04/15] Fix getRollup API --- models/interfaces.ts | 2 +- server/models/interfaces.ts | 6 +++--- server/services/RollupService.ts | 28 ++++++++++++---------------- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/models/interfaces.ts b/models/interfaces.ts index f797a761..fe67645e 100644 --- a/models/interfaces.ts +++ b/models/interfaces.ts @@ -58,7 +58,7 @@ export interface DocumentPolicy { policy: Policy; } -//Data model that contains both rollup item and metadata of rollup job +// Data model that contains both rollup item and metadata of rollup job export interface DocumentRollup { _id: string; _seqNo: number; diff --git a/server/models/interfaces.ts b/server/models/interfaces.ts index 3a4c9a35..5b2864aa 100644 --- a/server/models/interfaces.ts +++ b/server/models/interfaces.ts @@ -71,8 +71,8 @@ export interface GetFieldsResponse { } export interface GetRollupsResponse { - rollups: DocumentRollup[]; - totalRollups: number | undefined; + rollups: DocumentRollup; + totalRollups: number; metadata: any; } @@ -80,7 +80,7 @@ export interface PutRollupResponse { _id: string; _primary_term: number; _seq_no: number; - rollup: { rollup: RollupItem }; + rollup: { rollup: Rollup }; } export interface IndexUpdateResponse { diff --git a/server/services/RollupService.ts b/server/services/RollupService.ts index 6d595858..344acb40 100644 --- a/server/services/RollupService.ts +++ b/server/services/RollupService.ts @@ -15,17 +15,8 @@ import _ from "lodash"; import { IClusterClient, KibanaRequest, KibanaResponseFactory, IKibanaResponse, ResponseError, RequestHandlerContext } from "kibana/server"; -import { CLUSTER, INDEX } from "../utils/constants"; -import { - DeleteRollupParams, - DeleteRollupResponse, - GetRollupsResponse, - PutRollupParams, - PutRollupResponse, - SearchResponse, -} from "../models/interfaces"; -import { getMustQuery } from "../utils/helpers"; -import { RollupsSort, ServerResponse } from "../models/types"; +import { DeleteRollupParams, DeleteRollupResponse, GetRollupsResponse, PutRollupParams, PutRollupResponse } from "../models/interfaces"; +import { ServerResponse } from "../models/types"; import { DocumentRollup, Rollup, RollupMetadata } from "../../models/interfaces"; export default class RollupService { @@ -193,7 +184,7 @@ export default class RollupService { const { id } = request.params as { id: string }; const params = { rollupId: id }; const { callAsCurrentUser: callWithRequest } = this.esDriver.asScoped(request); - const getResponse = await callWithRequest("ism.getRollups", params); + const getResponse = await callWithRequest("ism.getRollup", params); const metadata = await callWithRequest("ism.explainRollup", params); const rollup = _.get(getResponse, "rollup", null); const seqNo = _.get(getResponse, "_seq_no"); @@ -345,11 +336,13 @@ export default class RollupService { metadata: null, })); + // Call getExplain if any rollup job exists if (totalRollups) { - const ids = rollups.map((rollup) => rollup._id).join(","); + // Concat rollup job ids + const ids = rollups.map((rollup: DocumentRollup) => rollup._id).join(","); const explainResponse = await this.explainRollup(context, request, response, ids); - if (explainResponse.payload.ok) { - rollups.map((item) => { + if (explainResponse.payload !== undefined && explainResponse.payload.ok) { + rollups.map((item: DocumentRollup) => { item.metadata = explainResponse.payload.response[item._id]; }); return response.custom({ @@ -359,7 +352,10 @@ export default class RollupService { } else return response.custom({ statusCode: 200, - body: { ok: false, error: explainResponse.payload.error }, + body: { + ok: false, + error: explainResponse.payload ? explainResponse.payload.error : "An error occurred when calling getExplain API.", + }, }); } return response.custom({ From 39ed1ab951cb6cd580d71da90d9eb38d806f10c3 Mon Sep 17 00:00:00 2001 From: Annie Date: Tue, 29 Dec 2020 15:08:00 -0800 Subject: [PATCH 05/15] Update RollupService.ts --- server/services/RollupService.ts | 36 +++++++++++++++++--------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/server/services/RollupService.ts b/server/services/RollupService.ts index 344acb40..92395ae3 100644 --- a/server/services/RollupService.ts +++ b/server/services/RollupService.ts @@ -339,24 +339,26 @@ export default class RollupService { // Call getExplain if any rollup job exists if (totalRollups) { // Concat rollup job ids - const ids = rollups.map((rollup: DocumentRollup) => rollup._id).join(","); + const ids = rollups.map((rollup) => rollup._id).join(","); + console.error(ids); const explainResponse = await this.explainRollup(context, request, response, ids); - if (explainResponse.payload !== undefined && explainResponse.payload.ok) { - rollups.map((item: DocumentRollup) => { - item.metadata = explainResponse.payload.response[item._id]; - }); - return response.custom({ - statusCode: 200, - body: { ok: true, response: { rollups: rollups, totalRollups: totalRollups, metadata: explainResponse } }, - }); - } else - return response.custom({ - statusCode: 200, - body: { - ok: false, - error: explainResponse.payload ? explainResponse.payload.error : "An error occurred when calling getExplain API.", - }, - }); + // if (explainResponse.payload.ok) { + console.log(explainResponse); + rollups.map((item) => { + item.metadata = explainResponse.payload.response[item._id]; + }); + return response.custom({ + statusCode: 200, + body: { ok: true, response: { rollups: rollups, totalRollups: totalRollups, metadata: explainResponse } }, + }); + // } else + // return response.custom({ + // statusCode: 200, + // body: { + // ok: false, + // error: explainResponse.payload ? explainResponse.payload.error : "An error occurred when calling getExplain API.", + // }, + // }); } return response.custom({ statusCode: 200, From f6000a05378da94d8458e561713faa317dbcb14c Mon Sep 17 00:00:00 2001 From: Annie Date: Tue, 29 Dec 2020 15:23:00 -0800 Subject: [PATCH 06/15] Reverting some changes to getRollups --- server/models/interfaces.ts | 2 +- server/services/RollupService.ts | 34 ++++++++++++++++---------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/server/models/interfaces.ts b/server/models/interfaces.ts index 5b2864aa..00cb4f9c 100644 --- a/server/models/interfaces.ts +++ b/server/models/interfaces.ts @@ -71,7 +71,7 @@ export interface GetFieldsResponse { } export interface GetRollupsResponse { - rollups: DocumentRollup; + rollups: DocumentRollup[]; totalRollups: number; metadata: any; } diff --git a/server/services/RollupService.ts b/server/services/RollupService.ts index 92395ae3..e07fbe75 100644 --- a/server/services/RollupService.ts +++ b/server/services/RollupService.ts @@ -342,23 +342,23 @@ export default class RollupService { const ids = rollups.map((rollup) => rollup._id).join(","); console.error(ids); const explainResponse = await this.explainRollup(context, request, response, ids); - // if (explainResponse.payload.ok) { - console.log(explainResponse); - rollups.map((item) => { - item.metadata = explainResponse.payload.response[item._id]; - }); - return response.custom({ - statusCode: 200, - body: { ok: true, response: { rollups: rollups, totalRollups: totalRollups, metadata: explainResponse } }, - }); - // } else - // return response.custom({ - // statusCode: 200, - // body: { - // ok: false, - // error: explainResponse.payload ? explainResponse.payload.error : "An error occurred when calling getExplain API.", - // }, - // }); + if (explainResponse.payload.ok) { + console.log(explainResponse); + rollups.map((item) => { + item.metadata = explainResponse.payload.response[item._id]; + }); + return response.custom({ + statusCode: 200, + body: { ok: true, response: { rollups: rollups, totalRollups: totalRollups, metadata: explainResponse } }, + }); + } else + return response.custom({ + statusCode: 200, + body: { + ok: false, + error: explainResponse.payload ? explainResponse.payload.error : "An error occurred when calling getExplain API.", + }, + }); } return response.custom({ statusCode: 200, From 66b8edd16bbb18e15ad3c1a780eb09695b5ab619 Mon Sep 17 00:00:00 2001 From: Annie Date: Mon, 4 Jan 2021 10:40:43 -0800 Subject: [PATCH 07/15] Modify a few variable name to match with backend response --- server/models/interfaces.ts | 2 +- server/services/RollupService.ts | 59 ++++++-------------------------- 2 files changed, 11 insertions(+), 50 deletions(-) diff --git a/server/models/interfaces.ts b/server/models/interfaces.ts index 00cb4f9c..81c70e3c 100644 --- a/server/models/interfaces.ts +++ b/server/models/interfaces.ts @@ -72,7 +72,7 @@ export interface GetFieldsResponse { export interface GetRollupsResponse { rollups: DocumentRollup[]; - totalRollups: number; + total_rollups: number; metadata: any; } diff --git a/server/services/RollupService.ts b/server/services/RollupService.ts index e07fbe75..997809ac 100644 --- a/server/services/RollupService.ts +++ b/server/services/RollupService.ts @@ -263,46 +263,9 @@ export default class RollupService { }); } }; - - explainRollup = async ( - context: RequestHandlerContext, - request: KibanaRequest, - response: KibanaResponseFactory, - idParams: string - ): Promise>> => { - try { - const params = { rollupId: idParams }; - const { callAsCurrentUser: callWithRequest } = this.esDriver.asScoped(request); - const rollupMetadata = await callWithRequest("ism.explainRollup", params); - if (rollupMetadata) { - return response.custom({ - statusCode: 200, - body: { - ok: true, - response: rollupMetadata, - }, - }); - } else { - return response.custom({ - statusCode: 200, - body: { - ok: false, - error: "Failed to load rollup metadata", - }, - }); - } - } catch (err) { - console.error("Index Management - RollupService - explainRollup:", err); - return response.custom({ - statusCode: 200, - body: { - ok: false, - error: "Explain rollup: " + err.message, - }, - }); - } - }; - + /** + * Performs a fuzzy search request on rollup id + */ getRollups = async ( context: RequestHandlerContext, request: KibanaRequest, @@ -327,7 +290,7 @@ export default class RollupService { const { callAsCurrentUser: callWithRequest } = this.esDriver.asScoped(request); const getRollupResponse: GetRollupsResponse = await callWithRequest("ism.getRollups", params); - const totalRollups = getRollupResponse.totalRollups; + const totalRollups = getRollupResponse.total_rollups; const rollups = getRollupResponse.rollups.map((rollup) => ({ _seqNo: rollup._seqNo as number, _primaryTerm: rollup._primaryTerm as number, @@ -340,29 +303,27 @@ export default class RollupService { if (totalRollups) { // Concat rollup job ids const ids = rollups.map((rollup) => rollup._id).join(","); - console.error(ids); - const explainResponse = await this.explainRollup(context, request, response, ids); - if (explainResponse.payload.ok) { - console.log(explainResponse); + const explainResponse = await callWithRequest("ism.explainRollup", { rollupId: ids }); + if (!explainResponse.error) { rollups.map((item) => { - item.metadata = explainResponse.payload.response[item._id]; + item.metadata = explainResponse[item._id]; }); return response.custom({ statusCode: 200, - body: { ok: true, response: { rollups: rollups, totalRollups: totalRollups, metadata: explainResponse } }, + body: { ok: true, response: { rollups: rollups, total_rollups: totalRollups, metadata: explainResponse } }, }); } else return response.custom({ statusCode: 200, body: { ok: false, - error: explainResponse.payload ? explainResponse.payload.error : "An error occurred when calling getExplain API.", + error: explainResponse ? explainResponse.error : "An error occurred when calling getExplain API.", }, }); } return response.custom({ statusCode: 200, - body: { ok: true, response: { rollups: rollups, totalRollups: totalRollups, metadata: {} } }, + body: { ok: true, response: { rollups: rollups, total_rollups: totalRollups, metadata: {} } }, }); } catch (err) { if (err.statusCode === 404 && err.body.error.type === "index_not_found_exception") { From 6dfe04a0c5503495e956976b8af8451de8848c20 Mon Sep 17 00:00:00 2001 From: Annie Date: Mon, 4 Jan 2021 10:42:56 -0800 Subject: [PATCH 08/15] Update RollupService.ts --- server/services/RollupService.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/services/RollupService.ts b/server/services/RollupService.ts index 997809ac..2315b89b 100644 --- a/server/services/RollupService.ts +++ b/server/services/RollupService.ts @@ -17,7 +17,7 @@ import _ from "lodash"; import { IClusterClient, KibanaRequest, KibanaResponseFactory, IKibanaResponse, ResponseError, RequestHandlerContext } from "kibana/server"; import { DeleteRollupParams, DeleteRollupResponse, GetRollupsResponse, PutRollupParams, PutRollupResponse } from "../models/interfaces"; import { ServerResponse } from "../models/types"; -import { DocumentRollup, Rollup, RollupMetadata } from "../../models/interfaces"; +import { DocumentRollup, Rollup } from "../../models/interfaces"; export default class RollupService { esDriver: IClusterClient; @@ -295,7 +295,7 @@ export default class RollupService { _seqNo: rollup._seqNo as number, _primaryTerm: rollup._primaryTerm as number, _id: rollup._id, - rollup: rollup, + rollup: rollup.rollup, metadata: null, })); @@ -329,7 +329,7 @@ export default class RollupService { if (err.statusCode === 404 && err.body.error.type === "index_not_found_exception") { return response.custom({ statusCode: 200, - body: { ok: true, response: { rollups: [], totalRollups: 0, metadata: null } }, + body: { ok: true, response: { rollups: [], total_rollups: 0, metadata: null } }, }); } console.error("Index Management - RollupService - getRollups", err); From ea42437bfe264a52e04bcedc7b144a4256a1dee1 Mon Sep 17 00:00:00 2001 From: Annie Date: Tue, 5 Jan 2021 14:35:14 -0800 Subject: [PATCH 09/15] Style fix --- server/models/interfaces.ts | 2 +- server/services/RollupService.ts | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/server/models/interfaces.ts b/server/models/interfaces.ts index 81c70e3c..00cb4f9c 100644 --- a/server/models/interfaces.ts +++ b/server/models/interfaces.ts @@ -72,7 +72,7 @@ export interface GetFieldsResponse { export interface GetRollupsResponse { rollups: DocumentRollup[]; - total_rollups: number; + totalRollups: number; metadata: any; } diff --git a/server/services/RollupService.ts b/server/services/RollupService.ts index 2315b89b..9f94cd35 100644 --- a/server/services/RollupService.ts +++ b/server/services/RollupService.ts @@ -289,9 +289,9 @@ export default class RollupService { }; const { callAsCurrentUser: callWithRequest } = this.esDriver.asScoped(request); - const getRollupResponse: GetRollupsResponse = await callWithRequest("ism.getRollups", params); + const getRollupResponse = await callWithRequest("ism.getRollups", params); const totalRollups = getRollupResponse.total_rollups; - const rollups = getRollupResponse.rollups.map((rollup) => ({ + const rollups = getRollupResponse.rollups.map((rollup: DocumentRollup) => ({ _seqNo: rollup._seqNo as number, _primaryTerm: rollup._primaryTerm as number, _id: rollup._id, @@ -302,15 +302,15 @@ export default class RollupService { // Call getExplain if any rollup job exists if (totalRollups) { // Concat rollup job ids - const ids = rollups.map((rollup) => rollup._id).join(","); + const ids = rollups.map((rollup: DocumentRollup) => rollup._id).join(","); const explainResponse = await callWithRequest("ism.explainRollup", { rollupId: ids }); if (!explainResponse.error) { - rollups.map((item) => { - item.metadata = explainResponse[item._id]; + rollups.map((rollup: DocumentRollup) => { + rollup.metadata = explainResponse[rollup._id]; }); return response.custom({ statusCode: 200, - body: { ok: true, response: { rollups: rollups, total_rollups: totalRollups, metadata: explainResponse } }, + body: { ok: true, response: { rollups: rollups, totalRollups: totalRollups, metadata: explainResponse } }, }); } else return response.custom({ @@ -323,13 +323,13 @@ export default class RollupService { } return response.custom({ statusCode: 200, - body: { ok: true, response: { rollups: rollups, total_rollups: totalRollups, metadata: {} } }, + body: { ok: true, response: { rollups: rollups, totalRollups: totalRollups, metadata: {} } }, }); } catch (err) { if (err.statusCode === 404 && err.body.error.type === "index_not_found_exception") { return response.custom({ statusCode: 200, - body: { ok: true, response: { rollups: [], total_rollups: 0, metadata: null } }, + body: { ok: true, response: { rollups: [], totalRollups: 0, metadata: null } }, }); } console.error("Index Management - RollupService - getRollups", err); From 25b59905b9cdbcb492b9b5cbd34ff435a9231a8f Mon Sep 17 00:00:00 2001 From: Annie Date: Sat, 16 Jan 2021 11:13:18 +0800 Subject: [PATCH 10/15] Fix getRollups --- .../Rollups/containers/Rollups/Rollups.tsx | 8 +++---- public/pages/Rollups/utils/constants.tsx | 2 +- server/clusters/ism/ismPlugin.ts | 21 ------------------- server/services/RollupService.ts | 17 ++++++++++----- 4 files changed, 17 insertions(+), 31 deletions(-) diff --git a/public/pages/Rollups/containers/Rollups/Rollups.tsx b/public/pages/Rollups/containers/Rollups/Rollups.tsx index 8b4023fb..92ff7d9c 100644 --- a/public/pages/Rollups/containers/Rollups/Rollups.tsx +++ b/public/pages/Rollups/containers/Rollups/Rollups.tsx @@ -342,14 +342,14 @@ export default class Rollups extends Component { render: (_id) => this.props.history.push(`${ROUTES.ROLLUP_DETAILS}?id=${_id}`)}>{_id}, }, { - field: "rollup.rollup.source_index", + field: "rollup.source_index", name: "Source index", sortable: true, textOnly: true, truncateText: true, }, { - field: "rollup.rollup.target_index", + field: "rollup.target_index", name: "Target index", sortable: true, textOnly: true, @@ -374,7 +374,7 @@ export default class Rollups extends Component { { field: "metadata.rollup_metadata.continuous", name: "Next rollup window", - sortable: true, + sortable: false, textOnly: true, render: (metadata) => metadata == null ? "-" : renderTime(metadata.next_window_start_time) + " - " + renderTime(metadata.next_window_end_time), @@ -382,7 +382,7 @@ export default class Rollups extends Component { { field: "metadata", name: "Rollup job status", - sortable: true, + sortable: false, textOnly: true, render: (metadata) => renderStatus(metadata), }, diff --git a/public/pages/Rollups/utils/constants.tsx b/public/pages/Rollups/utils/constants.tsx index 9776ae11..222d0aff 100644 --- a/public/pages/Rollups/utils/constants.tsx +++ b/public/pages/Rollups/utils/constants.tsx @@ -21,6 +21,6 @@ export const DEFAULT_QUERY_PARAMS = { from: 0, size: 20, search: "", - sortField: "name", + sortField: "_id", sortDirection: SortDirection.DESC, }; diff --git a/server/clusters/ism/ismPlugin.ts b/server/clusters/ism/ismPlugin.ts index 2d942308..27ae08af 100644 --- a/server/clusters/ism/ismPlugin.ts +++ b/server/clusters/ism/ismPlugin.ts @@ -174,27 +174,6 @@ export default function ismPlugin(Client: any, config: any, components: any) { ism.getRollups = ca({ url: { fmt: `${API.ROLLUP_JOBS_BASE}`, - req: { - from: { - type: "number", - required: true, - }, - size: { - type: "number", - required: true, - }, - search: { - type: "string", - }, - sortField: { - type: "string", - required: true, - }, - sortDirection: { - type: "string", - required: true, - }, - }, }, method: "GET", }); diff --git a/server/services/RollupService.ts b/server/services/RollupService.ts index 9f94cd35..2fa902da 100644 --- a/server/services/RollupService.ts +++ b/server/services/RollupService.ts @@ -273,18 +273,25 @@ export default class RollupService { ): Promise>> => { try { const { from, size, search, sortDirection, sortField } = request.query as { - from: number; - size: number; + from: string; + size: string; search: string; sortDirection: string; sortField: string; }; + const rollupSortMap: { [key: string]: string } = { + _id: "rollup.rollup_id.keyword", + "rollup.source_index": "rollup.source_index.keyword", + "rollup.target_index": "rollup.target_index.keyword", + "rollup.rollup.enabled": "rollup.enabled", + }; + const params = { - from, - size, + from: parseInt(from, 10), + size: parseInt(size, 10), search, - sortField, + sortField: rollupSortMap[sortField] || rollupSortMap._id, sortDirection, }; From f5075257f09c4a785a53f94a8b850b5e34a478e2 Mon Sep 17 00:00:00 2001 From: Annie Date: Tue, 26 Jan 2021 21:05:43 +0800 Subject: [PATCH 11/15] Remove unused imports --- .../ChangeManagedIndices/ChangeManagedIndices.test.tsx | 1 - .../containers/RollupDetails/RollupDetails.tsx | 9 +-------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/public/pages/ChangePolicy/components/ChangeManagedIndices/ChangeManagedIndices.test.tsx b/public/pages/ChangePolicy/components/ChangeManagedIndices/ChangeManagedIndices.test.tsx index 86e597e6..5e60a8d7 100644 --- a/public/pages/ChangePolicy/components/ChangeManagedIndices/ChangeManagedIndices.test.tsx +++ b/public/pages/ChangePolicy/components/ChangeManagedIndices/ChangeManagedIndices.test.tsx @@ -20,7 +20,6 @@ import ChangeManagedIndices from "./ChangeManagedIndices"; import { browserServicesMock } from "../../../../../test/mocks"; import coreServicesMock from "../../../../../test/mocks/coreServicesMock"; import { CoreServicesContext } from "../../../../components/core_services"; -import RetryModal from "../../../ManagedIndices/components/RetryModal"; describe(" spec", () => { it("renders the component", async () => { diff --git a/public/pages/RollupDetails/containers/RollupDetails/RollupDetails.tsx b/public/pages/RollupDetails/containers/RollupDetails/RollupDetails.tsx index c39f26f6..7554b466 100644 --- a/public/pages/RollupDetails/containers/RollupDetails/RollupDetails.tsx +++ b/public/pages/RollupDetails/containers/RollupDetails/RollupDetails.tsx @@ -39,14 +39,7 @@ import GeneralInformation from "../../components/GeneralInformation/GeneralInfor import RollupStatus from "../../components/RollupStatus/RollupStatus"; import AggregationAndMetricsSettings from "../../components/AggregationAndMetricsSettings/AggregationAndMetricsSettings"; import { parseTimeunit } from "../../../CreateRollup/utils/helpers"; -import { - DimensionItem, - MetricItem, - RollupDimensionItem, - RollupMetadata, - RollupMetricItem, - DateHistogramItem, -} from "../../../../../models/interfaces"; +import { DimensionItem, MetricItem, RollupDimensionItem, RollupMetadata, RollupMetricItem } from "../../../../../models/interfaces"; import { renderTime } from "../../../Rollups/utils/helpers"; import DeleteModal from "../../../Rollups/components/DeleteModal"; import { CoreServicesContext } from "../../../../components/core_services"; From b1eba6fdf791b2cdc310d6704f0d6fd9a0289337 Mon Sep 17 00:00:00 2001 From: Annie Date: Tue, 26 Jan 2021 22:23:08 +0800 Subject: [PATCH 12/15] Update Rollups.test.tsx.snap --- .../__snapshots__/Rollups.test.tsx.snap | 61 ++++++------------- 1 file changed, 18 insertions(+), 43 deletions(-) diff --git a/public/pages/Rollups/containers/Rollups/__snapshots__/Rollups.test.tsx.snap b/public/pages/Rollups/containers/Rollups/__snapshots__/Rollups.test.tsx.snap index 916f3d15..be63148f 100644 --- a/public/pages/Rollups/containers/Rollups/__snapshots__/Rollups.test.tsx.snap +++ b/public/pages/Rollups/containers/Rollups/__snapshots__/Rollups.test.tsx.snap @@ -261,14 +261,14 @@ exports[` spec renders the component 1`] = ` - + - + From cf7f6d8d1bb6a5128afa2d635eb5bc26c220defe Mon Sep 17 00:00:00 2001 From: Annie Date: Tue, 26 Jan 2021 22:36:35 +0800 Subject: [PATCH 13/15] Add getRollups-fix branch to run cypress and unit tests workflow To show that unit tests and cypress tests will pass. Will remove upon success. --- .github/workflows/cypress-workflow.yml | 1 + .github/workflows/unit-tests-workflow.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/cypress-workflow.yml b/.github/workflows/cypress-workflow.yml index 4af087c3..02f8da76 100644 --- a/.github/workflows/cypress-workflow.yml +++ b/.github/workflows/cypress-workflow.yml @@ -3,6 +3,7 @@ on: push: branches: - master + - getRollups-fix jobs: tests: diff --git a/.github/workflows/unit-tests-workflow.yml b/.github/workflows/unit-tests-workflow.yml index bf8b42a8..2f7d0998 100644 --- a/.github/workflows/unit-tests-workflow.yml +++ b/.github/workflows/unit-tests-workflow.yml @@ -3,6 +3,7 @@ on: push: branches: - master + - getRollups-fix jobs: tests: From 7c8cefdc382020a9cb062fa09f36dea134199a80 Mon Sep 17 00:00:00 2001 From: Annie Date: Wed, 27 Jan 2021 05:22:44 +0800 Subject: [PATCH 14/15] Add more timeout to install sample data --- cypress/integration/rollups_spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cypress/integration/rollups_spec.js b/cypress/integration/rollups_spec.js index a97ea310..e3abb2d1 100644 --- a/cypress/integration/rollups_spec.js +++ b/cypress/integration/rollups_spec.js @@ -38,7 +38,7 @@ describe("Rollups", () => { cy.get(`button[data-test-subj="addSampleDataSetecommerce"]`).click({ force: true }); // Verify that sample data is add by checking toast notification - cy.contains("Sample eCommerce orders installed"); + cy.contains("Sample eCommerce orders installed", { timeout: 60000 }); // Visit ISM Kibana cy.visit(`${Cypress.env("kibana")}/app/${PLUGIN_NAME}#/rollups`); From dab879565a54e83a43f03480f6143c0c3c1ab08b Mon Sep 17 00:00:00 2001 From: Annie Date: Wed, 27 Jan 2021 13:01:57 +0800 Subject: [PATCH 15/15] Remove getRollups-fix from workflow yml files --- .github/workflows/cypress-workflow.yml | 1 - .github/workflows/unit-tests-workflow.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/cypress-workflow.yml b/.github/workflows/cypress-workflow.yml index 02f8da76..4af087c3 100644 --- a/.github/workflows/cypress-workflow.yml +++ b/.github/workflows/cypress-workflow.yml @@ -3,7 +3,6 @@ on: push: branches: - master - - getRollups-fix jobs: tests: diff --git a/.github/workflows/unit-tests-workflow.yml b/.github/workflows/unit-tests-workflow.yml index 2f7d0998..bf8b42a8 100644 --- a/.github/workflows/unit-tests-workflow.yml +++ b/.github/workflows/unit-tests-workflow.yml @@ -3,7 +3,6 @@ on: push: branches: - master - - getRollups-fix jobs: tests: