From bf4f4f0afeec681379a45d92e97cdbb200059e75 Mon Sep 17 00:00:00 2001 From: Poff Poffenberger Date: Mon, 16 Dec 2019 09:25:09 -0600 Subject: [PATCH] Removing old usage collector fom legacy Canvas plugin --- x-pack/legacy/plugins/canvas/server/plugin.ts | 2 - .../plugins/canvas/server/usage/collector.ts | 50 ----- .../canvas/server/usage/collector_helpers.ts | 33 ---- .../usage/custom_element_collector.test.ts | 84 -------- .../server/usage/custom_element_collector.ts | 121 ------------ .../plugins/canvas/server/usage/index.ts | 7 - .../server/usage/workpad_collector.test.ts | 86 --------- .../canvas/server/usage/workpad_collector.ts | 181 ------------------ .../custom_element_collector.test.ts | 2 +- .../collectors/workpad_collector.test.ts | 2 +- x-pack/plugins/canvas/server/plugin.ts | 1 + 11 files changed, 3 insertions(+), 566 deletions(-) delete mode 100644 x-pack/legacy/plugins/canvas/server/usage/collector.ts delete mode 100644 x-pack/legacy/plugins/canvas/server/usage/collector_helpers.ts delete mode 100644 x-pack/legacy/plugins/canvas/server/usage/custom_element_collector.test.ts delete mode 100644 x-pack/legacy/plugins/canvas/server/usage/custom_element_collector.ts delete mode 100644 x-pack/legacy/plugins/canvas/server/usage/index.ts delete mode 100644 x-pack/legacy/plugins/canvas/server/usage/workpad_collector.test.ts delete mode 100644 x-pack/legacy/plugins/canvas/server/usage/workpad_collector.ts diff --git a/x-pack/legacy/plugins/canvas/server/plugin.ts b/x-pack/legacy/plugins/canvas/server/plugin.ts index b338971103381..d347398ed02af 100644 --- a/x-pack/legacy/plugins/canvas/server/plugin.ts +++ b/x-pack/legacy/plugins/canvas/server/plugin.ts @@ -7,7 +7,6 @@ import { CoreSetup, PluginsSetup } from './shim'; import { routes } from './routes'; import { functions } from '../canvas_plugin_src/functions/server'; -import { registerCanvasUsageCollector } from './usage'; import { loadSampleData } from './sample_data'; export class Plugin { @@ -61,7 +60,6 @@ export class Plugin { }, }); - registerCanvasUsageCollector(plugins.usageCollection, core); loadSampleData( plugins.sampleData.addSavedObjectsToSampleDataset, plugins.sampleData.addAppLinksToSampleDataset diff --git a/x-pack/legacy/plugins/canvas/server/usage/collector.ts b/x-pack/legacy/plugins/canvas/server/usage/collector.ts deleted file mode 100644 index ae009f9265722..0000000000000 --- a/x-pack/legacy/plugins/canvas/server/usage/collector.ts +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { CallCluster } from 'src/legacy/core_plugins/elasticsearch'; -import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; -import { CoreSetup } from '../shim'; -// @ts-ignore missing local declaration -import { CANVAS_USAGE_TYPE } from '../../common/lib/constants'; -import { workpadCollector } from './workpad_collector'; -import { customElementCollector } from './custom_element_collector'; -import { TelemetryCollector } from '../../types'; - -const collectors: TelemetryCollector[] = [workpadCollector, customElementCollector]; - -/* - Register the canvas usage collector function - - This will call all of the defined collectors and combine the individual results into a single object - to be returned to the caller. - - A usage collector function returns an object derived from current data in the ES Cluster. -*/ -export function registerCanvasUsageCollector( - usageCollection: UsageCollectionSetup, - core: CoreSetup -) { - const kibanaIndex = core.getServerConfig().get('kibana.index'); - const canvasCollector = usageCollection.makeUsageCollector({ - type: CANVAS_USAGE_TYPE, - isReady: () => true, - fetch: async (callCluster: CallCluster) => { - const collectorResults = await Promise.all( - collectors.map(collector => collector(kibanaIndex, callCluster)) - ); - - return collectorResults.reduce( - (reduction, usage) => { - return { ...reduction, ...usage }; - }, - - {} - ); - }, - }); - - usageCollection.registerCollector(canvasCollector); -} diff --git a/x-pack/legacy/plugins/canvas/server/usage/collector_helpers.ts b/x-pack/legacy/plugins/canvas/server/usage/collector_helpers.ts deleted file mode 100644 index 784042fb4d94d..0000000000000 --- a/x-pack/legacy/plugins/canvas/server/usage/collector_helpers.ts +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -/* - * @param ast: an ast that includes functions to track - * @param cb: callback to do something with a function that has been found - */ - -import { ExpressionAST, ExpressionArgAST } from '../../types'; - -function isExpression(maybeExpression: ExpressionArgAST): maybeExpression is ExpressionAST { - return typeof maybeExpression === 'object'; -} - -export function collectFns(ast: ExpressionArgAST, cb: (functionName: string) => void) { - if (isExpression(ast)) { - ast.chain.forEach(({ function: cFunction, arguments: cArguments }) => { - cb(cFunction); - - // recurse the arguments and update the set along the way - Object.keys(cArguments).forEach(argName => { - cArguments[argName].forEach(subAst => { - if (subAst != null) { - collectFns(subAst, cb); - } - }); - }); - }); - } -} diff --git a/x-pack/legacy/plugins/canvas/server/usage/custom_element_collector.test.ts b/x-pack/legacy/plugins/canvas/server/usage/custom_element_collector.test.ts deleted file mode 100644 index f09bb704b09e3..0000000000000 --- a/x-pack/legacy/plugins/canvas/server/usage/custom_element_collector.test.ts +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { summarizeCustomElements } from './custom_element_collector'; -import { TelemetryCustomElementDocument } from '../../types'; - -function mockCustomElement(...nodeExpressions: string[]): TelemetryCustomElementDocument { - return { - content: JSON.stringify({ - selectedNodes: nodeExpressions.map(expression => ({ - expression, - })), - }), - }; -} - -describe('custom_element_collector.handleResponse', () => { - describe('invalid responses', () => { - it('returns nothing if no valid hits', () => { - expect(summarizeCustomElements([])).toEqual({}); - }); - - it('returns nothing if no valid elements', () => { - const customElements = [ - { - content: 'invalid json', - }, - ]; - - expect(summarizeCustomElements(customElements)).toEqual({}); - }); - }); - - it('counts total custom elements', () => { - const elements = [mockCustomElement(''), mockCustomElement('')]; - - const data = summarizeCustomElements(elements); - expect(data.custom_elements).not.toBe(null); - - if (data.custom_elements) { - expect(data.custom_elements.count).toEqual(elements.length); - } - }); - - it('reports all the functions used in custom elements', () => { - const functions1 = ['a', 'b', 'c']; - const functions2 = ['c', 'd', 'e', 'f']; - const expectedFunctions = Array.from(new Set([...functions1, ...functions2])); - - const elements = [mockCustomElement(functions1.join('|')), mockCustomElement(...functions2)]; - - const data = summarizeCustomElements(elements); - expect(data.custom_elements).not.toBe(null); - - if (data.custom_elements) { - expect(data.custom_elements.functions_in_use).toEqual(expectedFunctions); - } - }); - - it('reports minimum, maximum, and avg elements in a custom element', () => { - const functionsMin = ['a', 'b', 'c']; - const functionsMax = ['d', 'e', 'f', 'g', 'h']; - const functionsOther = ['i', 'j', 'k', 'l']; - const avgFunctions = (functionsMin.length + functionsMax.length + functionsOther.length) / 3; - - const elements = [ - mockCustomElement(...functionsMin), - mockCustomElement(...functionsMax), - mockCustomElement(...functionsOther), - ]; - - const result = summarizeCustomElements(elements); - expect(result.custom_elements).not.toBe(null); - - if (result.custom_elements) { - expect(result.custom_elements.elements.max).toEqual(functionsMax.length); - expect(result.custom_elements.elements.min).toEqual(functionsMin.length); - expect(result.custom_elements.elements.avg).toEqual(avgFunctions); - } - }); -}); diff --git a/x-pack/legacy/plugins/canvas/server/usage/custom_element_collector.ts b/x-pack/legacy/plugins/canvas/server/usage/custom_element_collector.ts deleted file mode 100644 index 218ac0fed08c9..0000000000000 --- a/x-pack/legacy/plugins/canvas/server/usage/custom_element_collector.ts +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { SearchParams } from 'elasticsearch'; -import { get } from 'lodash'; -import { fromExpression } from '@kbn/interpreter/common'; -import { collectFns } from './collector_helpers'; -import { TelemetryCollector } from '../../types'; -import { ExpressionAST, TelemetryCustomElement, TelemetryCustomElementDocument } from '../../types'; - -const CUSTOM_ELEMENT_TYPE = 'canvas-element'; -interface CustomElementSearch { - [CUSTOM_ELEMENT_TYPE]: TelemetryCustomElementDocument; -} - -interface CustomElementTelemetry { - custom_elements?: { - count: number; - elements: { - min: number; - max: number; - avg: number; - }; - functions_in_use: string[]; - }; -} - -function isCustomElement(maybeCustomElement: any): maybeCustomElement is TelemetryCustomElement { - return ( - maybeCustomElement !== null && - Array.isArray(maybeCustomElement.selectedNodes) && - maybeCustomElement.selectedNodes.every( - (node: any) => node.expression && typeof node.expression === 'string' - ) - ); -} - -function parseJsonOrNull(maybeJson: string) { - try { - return JSON.parse(maybeJson); - } catch (e) { - return null; - } -} - -/** - Calculate statistics about a collection of CustomElement Documents - @param customElements - Array of CustomElement documents - @returns Statistics about how Custom Elements are being used -*/ -export function summarizeCustomElements( - customElements: TelemetryCustomElementDocument[] -): CustomElementTelemetry { - const functionSet = new Set(); - - const parsedContents: TelemetryCustomElement[] = customElements - .map(element => element.content) - .map(parseJsonOrNull) - .filter(isCustomElement); - - if (parsedContents.length === 0) { - return {}; - } - - const elements = { - min: Infinity, - max: -Infinity, - avg: 0, - }; - - let totalElements = 0; - - parsedContents.map(contents => { - contents.selectedNodes.map(node => { - const ast: ExpressionAST = fromExpression(node.expression) as ExpressionAST; // TODO: Remove once fromExpression is properly typed - collectFns(ast, (cFunction: string) => { - functionSet.add(cFunction); - }); - }); - elements.min = Math.min(elements.min, contents.selectedNodes.length); - elements.max = Math.max(elements.max, contents.selectedNodes.length); - totalElements += contents.selectedNodes.length; - }); - - elements.avg = totalElements / parsedContents.length; - - return { - custom_elements: { - elements, - count: customElements.length, - functions_in_use: Array.from(functionSet), - }, - }; -} - -const customElementCollector: TelemetryCollector = async function customElementCollector( - kibanaIndex, - callCluster -) { - const customElementParams: SearchParams = { - size: 10000, - index: kibanaIndex, - ignoreUnavailable: true, - filterPath: [`hits.hits._source.${CUSTOM_ELEMENT_TYPE}.content`], - body: { query: { bool: { filter: { term: { type: CUSTOM_ELEMENT_TYPE } } } } }, - }; - - const esResponse = await callCluster('search', customElementParams); - - if (get(esResponse, 'hits.hits.length') > 0) { - const customElements = esResponse.hits.hits.map(hit => hit._source[CUSTOM_ELEMENT_TYPE]); - return summarizeCustomElements(customElements); - } - - return {}; -}; - -export { customElementCollector }; diff --git a/x-pack/legacy/plugins/canvas/server/usage/index.ts b/x-pack/legacy/plugins/canvas/server/usage/index.ts deleted file mode 100644 index b3d16a2b7be31..0000000000000 --- a/x-pack/legacy/plugins/canvas/server/usage/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -export { registerCanvasUsageCollector } from './collector'; diff --git a/x-pack/legacy/plugins/canvas/server/usage/workpad_collector.test.ts b/x-pack/legacy/plugins/canvas/server/usage/workpad_collector.test.ts deleted file mode 100644 index 420b785771bfe..0000000000000 --- a/x-pack/legacy/plugins/canvas/server/usage/workpad_collector.test.ts +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import clonedeep from 'lodash.clonedeep'; -import { summarizeWorkpads } from './workpad_collector'; -import { workpads } from '../../__tests__/fixtures/workpads'; - -describe('usage collector handle es response data', () => { - it('should summarize workpads, pages, and elements', () => { - const usage = summarizeWorkpads(workpads); - expect(usage).toEqual({ - workpads: { - total: 6, // num workpad documents in .kibana index - }, - pages: { - total: 16, // num pages in all the workpads - per_workpad: { avg: 2.6666666666666665, min: 1, max: 4 }, - }, - elements: { - total: 34, // num elements in all the pages - per_page: { avg: 2.125, min: 1, max: 5 }, - }, - functions: { - per_element: { avg: 4, min: 2, max: 7 }, - total: 36, - in_use: [ - 'demodata', - 'ply', - 'rowCount', - 'as', - 'staticColumn', - 'math', - 'mapColumn', - 'sort', - 'pointseries', - 'plot', - 'seriesStyle', - 'filters', - 'markdown', - 'render', - 'getCell', - 'repeatImage', - 'pie', - 'table', - 'image', - 'shape', - ], - }, - }); - }); - - it('should collect correctly if an expression has null as an argument (possible sub-expression)', () => { - const workpad = clonedeep(workpads[0]); - workpad.pages[0].elements[0].expression = 'toast butter=null'; - - const mockWorkpads = [workpad]; - const usage = summarizeWorkpads(mockWorkpads); - expect(usage).toEqual({ - workpads: { total: 1 }, - pages: { total: 1, per_workpad: { avg: 1, min: 1, max: 1 } }, - elements: { total: 1, per_page: { avg: 1, min: 1, max: 1 } }, - functions: { total: 1, in_use: ['toast'], per_element: { avg: 1, min: 1, max: 1 } }, - }); - }); - - it('should fail gracefully if workpad has 0 pages (corrupted workpad)', () => { - const workpad = clonedeep(workpads[0]); - workpad.pages = []; - const mockWorkpadsCorrupted = [workpad]; - const usage = summarizeWorkpads(mockWorkpadsCorrupted); - expect(usage).toEqual({ - workpads: { total: 1 }, - pages: { total: 0, per_workpad: { avg: 0, min: 0, max: 0 } }, - elements: undefined, - functions: undefined, - }); - }); - - it('should fail gracefully in general', () => { - const usage = summarizeWorkpads([]); - expect(usage).toEqual({}); - }); -}); diff --git a/x-pack/legacy/plugins/canvas/server/usage/workpad_collector.ts b/x-pack/legacy/plugins/canvas/server/usage/workpad_collector.ts deleted file mode 100644 index 5e6e2fa6dbd6a..0000000000000 --- a/x-pack/legacy/plugins/canvas/server/usage/workpad_collector.ts +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { SearchParams } from 'elasticsearch'; -import { sum as arraySum, min as arrayMin, max as arrayMax, get } from 'lodash'; -import { fromExpression } from '@kbn/interpreter/common'; -import { CANVAS_TYPE } from '../../common/lib/constants'; -import { collectFns } from './collector_helpers'; -import { ExpressionAST, TelemetryCollector, CanvasWorkpad } from '../../types'; - -interface WorkpadSearch { - [CANVAS_TYPE]: CanvasWorkpad; -} - -interface WorkpadTelemetry { - workpads?: { - total: number; - }; - pages?: { - total: number; - per_workpad: { - avg: number; - min: number; - max: number; - }; - }; - elements?: { - total: number; - per_page: { - avg: number; - min: number; - max: number; - }; - }; - functions?: { - total: number; - in_use: string[]; - per_element: { - avg: number; - min: number; - max: number; - }; - }; -} - -/** - Gather statistic about the given workpads - @param workpadDocs a collection of workpad documents - @returns Workpad Telemetry Data -*/ -export function summarizeWorkpads(workpadDocs: CanvasWorkpad[]): WorkpadTelemetry { - const functionSet = new Set(); - - if (workpadDocs.length === 0) { - return {}; - } - - // make a summary of info about each workpad - const workpadsInfo = workpadDocs.map(workpad => { - let pages = { count: 0 }; - try { - pages = { count: workpad.pages.length }; - } catch (err) { - // eslint-disable-next-line - console.warn(err, workpad); - } - const elementCounts = workpad.pages.reduce( - (accum, page) => accum.concat(page.elements.length), - [] - ); - const functionCounts = workpad.pages.reduce((accum, page) => { - return page.elements.map(element => { - const ast: ExpressionAST = fromExpression(element.expression) as ExpressionAST; // TODO: Remove once fromExpression is properly typed - collectFns(ast, cFunction => { - functionSet.add(cFunction); - }); - return ast.chain.length; // get the number of parts in the expression - }); - }, []); - - return { pages, elementCounts, functionCounts }; - }); - - // combine together info from across the workpads - const combinedWorkpadsInfo = workpadsInfo.reduce<{ - pageMin: number; - pageMax: number; - pageCounts: number[]; - elementCounts: number[]; - functionCounts: number[]; - }>( - (accum, pageInfo) => { - const { pages, elementCounts, functionCounts } = pageInfo; - - return { - pageMin: pages.count < accum.pageMin ? pages.count : accum.pageMin, - pageMax: pages.count > accum.pageMax ? pages.count : accum.pageMax, - pageCounts: accum.pageCounts.concat(pages.count), - elementCounts: accum.elementCounts.concat(elementCounts), - functionCounts: accum.functionCounts.concat(functionCounts), - }; - }, - { - pageMin: Infinity, - pageMax: -Infinity, - pageCounts: [], - elementCounts: [], - functionCounts: [], - } - ); - const { pageCounts, pageMin, pageMax, elementCounts, functionCounts } = combinedWorkpadsInfo; - - const pageTotal = arraySum(pageCounts); - const elementsTotal = arraySum(elementCounts); - const functionsTotal = arraySum(functionCounts); - const pagesInfo = - workpadsInfo.length > 0 - ? { - total: pageTotal, - per_workpad: { - avg: pageTotal / pageCounts.length, - min: pageMin, - max: pageMax, - }, - } - : undefined; - const elementsInfo = - pageTotal > 0 - ? { - total: elementsTotal, - per_page: { - avg: elementsTotal / elementCounts.length, - min: arrayMin(elementCounts), - max: arrayMax(elementCounts), - }, - } - : undefined; - const functionsInfo = - elementsTotal > 0 - ? { - total: functionsTotal, - in_use: Array.from(functionSet), - per_element: { - avg: functionsTotal / functionCounts.length, - min: arrayMin(functionCounts), - max: arrayMax(functionCounts), - }, - } - : undefined; - - return { - workpads: { total: workpadsInfo.length }, - pages: pagesInfo, - elements: elementsInfo, - functions: functionsInfo, - }; -} - -const workpadCollector: TelemetryCollector = async function(kibanaIndex, callCluster) { - const searchParams: SearchParams = { - size: 10000, // elasticsearch index.max_result_window default value - index: kibanaIndex, - ignoreUnavailable: true, - filterPath: ['hits.hits._source.canvas-workpad', '-hits.hits._source.canvas-workpad.assets'], - body: { query: { bool: { filter: { term: { type: CANVAS_TYPE } } } } }, - }; - - const esResponse = await callCluster('search', searchParams); - - if (get(esResponse, 'hits.hits.length') > 0) { - const workpads = esResponse.hits.hits.map(hit => hit._source[CANVAS_TYPE]); - return summarizeWorkpads(workpads); - } - - return {}; -}; - -export { workpadCollector }; diff --git a/x-pack/plugins/canvas/server/collectors/custom_element_collector.test.ts b/x-pack/plugins/canvas/server/collectors/custom_element_collector.test.ts index f09bb704b09e3..89b30e6968e70 100644 --- a/x-pack/plugins/canvas/server/collectors/custom_element_collector.test.ts +++ b/x-pack/plugins/canvas/server/collectors/custom_element_collector.test.ts @@ -5,7 +5,7 @@ */ import { summarizeCustomElements } from './custom_element_collector'; -import { TelemetryCustomElementDocument } from '../../types'; +import { TelemetryCustomElementDocument } from '../../../../legacy/plugins/canvas/types'; function mockCustomElement(...nodeExpressions: string[]): TelemetryCustomElementDocument { return { diff --git a/x-pack/plugins/canvas/server/collectors/workpad_collector.test.ts b/x-pack/plugins/canvas/server/collectors/workpad_collector.test.ts index 420b785771bfe..70bc074ff3df8 100644 --- a/x-pack/plugins/canvas/server/collectors/workpad_collector.test.ts +++ b/x-pack/plugins/canvas/server/collectors/workpad_collector.test.ts @@ -6,7 +6,7 @@ import clonedeep from 'lodash.clonedeep'; import { summarizeWorkpads } from './workpad_collector'; -import { workpads } from '../../__tests__/fixtures/workpads'; +import { workpads } from '../../../../legacy/plugins/canvas/__tests__/fixtures/workpads'; describe('usage collector handle es response data', () => { it('should summarize workpads, pages, and elements', () => { diff --git a/x-pack/plugins/canvas/server/plugin.ts b/x-pack/plugins/canvas/server/plugin.ts index 4cf88403a5ec2..0f27c68903b3d 100644 --- a/x-pack/plugins/canvas/server/plugin.ts +++ b/x-pack/plugins/canvas/server/plugin.ts @@ -25,6 +25,7 @@ export class CanvasPlugin implements Plugin { initRoutes({ router: canvasRouter, logger: this.logger }); + // we need the kibana index provided by global config for the Canvas usage collector const globalConfig = await this.initializerContext.config.legacy.globalConfig$ .pipe(first()) .toPromise();