From c251f2f0686295174765f9fcfd8ed8504d1c4df2 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 3 Mar 2020 11:08:27 +0100 Subject: [PATCH] [ML] Transforms: Migrate server plugin to NP. (#58714) (#59115) Migrate transform legacy server to NP. - Create server plugin/index for transform in x-pack/plugins. - Move all legacy/server files to plugins/transform --- x-pack/.i18nrc.json | 2 +- x-pack/legacy/plugins/transform/index.ts | 19 +- .../legacy/plugins/transform/server/plugin.ts | 17 - .../transform/server/routes/api/app.ts | 99 ----- .../server/routes/api/register_routes.ts | 14 - .../routes/api/transform_audit_messages.ts | 84 ---- .../transform/server/routes/api/transforms.ts | 261 ------------- .../legacy/plugins/transform/server/shim.ts | 46 --- x-pack/plugins/transform/kibana.json | 15 + .../server/client/elasticsearch_transform.ts | 0 x-pack/plugins/transform/server/index.ts | 11 + x-pack/plugins/transform/server/plugin.ts | 89 +++++ .../server/routes/api/error_utils.ts | 16 +- .../transform/server/routes/api/privileges.ts | 85 +++++ .../transform/server/routes/api/schema.ts | 16 + .../transform/server/routes/api/transforms.ts | 360 ++++++++++++++++++ .../routes/api/transforms_audit_messages.ts | 91 +++++ .../plugins/transform/server/routes/index.ts | 24 ++ .../transform/server/services/index.ts | 7 + .../transform/server/services/license.ts | 91 +++++ x-pack/plugins/transform/server/types.ts | 18 + 21 files changed, 824 insertions(+), 541 deletions(-) delete mode 100644 x-pack/legacy/plugins/transform/server/plugin.ts delete mode 100644 x-pack/legacy/plugins/transform/server/routes/api/app.ts delete mode 100644 x-pack/legacy/plugins/transform/server/routes/api/register_routes.ts delete mode 100644 x-pack/legacy/plugins/transform/server/routes/api/transform_audit_messages.ts delete mode 100644 x-pack/legacy/plugins/transform/server/routes/api/transforms.ts delete mode 100644 x-pack/legacy/plugins/transform/server/shim.ts create mode 100644 x-pack/plugins/transform/kibana.json rename x-pack/{legacy => }/plugins/transform/server/client/elasticsearch_transform.ts (100%) create mode 100644 x-pack/plugins/transform/server/index.ts create mode 100644 x-pack/plugins/transform/server/plugin.ts rename x-pack/{legacy => }/plugins/transform/server/routes/api/error_utils.ts (80%) create mode 100644 x-pack/plugins/transform/server/routes/api/privileges.ts create mode 100644 x-pack/plugins/transform/server/routes/api/schema.ts create mode 100644 x-pack/plugins/transform/server/routes/api/transforms.ts create mode 100644 x-pack/plugins/transform/server/routes/api/transforms_audit_messages.ts create mode 100644 x-pack/plugins/transform/server/routes/index.ts create mode 100644 x-pack/plugins/transform/server/services/index.ts create mode 100644 x-pack/plugins/transform/server/services/license.ts create mode 100644 x-pack/plugins/transform/server/types.ts diff --git a/x-pack/.i18nrc.json b/x-pack/.i18nrc.json index 51099815ec938..8f5a5ea4f10e4 100644 --- a/x-pack/.i18nrc.json +++ b/x-pack/.i18nrc.json @@ -39,7 +39,7 @@ "xpack.snapshotRestore": "legacy/plugins/snapshot_restore", "xpack.spaces": ["legacy/plugins/spaces", "plugins/spaces"], "xpack.taskManager": "legacy/plugins/task_manager", - "xpack.transform": "legacy/plugins/transform", + "xpack.transform": ["legacy/plugins/transform", "plugins/transform"], "xpack.triggersActionsUI": "plugins/triggers_actions_ui", "xpack.upgradeAssistant": "plugins/upgrade_assistant", "xpack.uptime": "legacy/plugins/uptime", diff --git a/x-pack/legacy/plugins/transform/index.ts b/x-pack/legacy/plugins/transform/index.ts index d0799f46cbd25..10f4732152c43 100644 --- a/x-pack/legacy/plugins/transform/index.ts +++ b/x-pack/legacy/plugins/transform/index.ts @@ -4,11 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import { Legacy } from 'kibana'; import { resolve } from 'path'; + import { PLUGIN } from './common/constants'; -import { Plugin as TransformPlugin } from './server/plugin'; -import { createServerShim } from './server/shim'; export function transform(kibana: any) { return new kibana.Plugin({ @@ -20,20 +18,5 @@ export function transform(kibana: any) { styleSheetPaths: resolve(__dirname, 'public/app/index.scss'), managementSections: ['plugins/transform'], }, - init(server: Legacy.Server) { - const { core, plugins } = createServerShim(server, PLUGIN.ID); - const transformPlugin = new TransformPlugin(); - - // Start plugin - transformPlugin.start(core, plugins); - - // Register license checker - plugins.license.registerLicenseChecker( - server, - PLUGIN.ID, - PLUGIN.getI18nName(), - PLUGIN.MINIMUM_LICENSE_REQUIRED - ); - }, }); } diff --git a/x-pack/legacy/plugins/transform/server/plugin.ts b/x-pack/legacy/plugins/transform/server/plugin.ts deleted file mode 100644 index f9264ee1f2507..0000000000000 --- a/x-pack/legacy/plugins/transform/server/plugin.ts +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -import { API_BASE_PATH } from '../common/constants'; -import { registerRoutes } from './routes/api/register_routes'; -import { Core, Plugins } from './shim'; - -export class Plugin { - public start(core: Core, plugins: Plugins): void { - const router = core.http.createRouter(API_BASE_PATH); - - // Register routes - registerRoutes(router, plugins); - } -} diff --git a/x-pack/legacy/plugins/transform/server/routes/api/app.ts b/x-pack/legacy/plugins/transform/server/routes/api/app.ts deleted file mode 100644 index c3189794b6eb0..0000000000000 --- a/x-pack/legacy/plugins/transform/server/routes/api/app.ts +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -import { Router, RouterRouteHandler } from '../../../../../server/lib/create_router'; -import { wrapCustomError } from '../../../../../server/lib/create_router/error_wrappers'; -import { APP_CLUSTER_PRIVILEGES, APP_INDEX_PRIVILEGES } from '../../../common/constants'; -// NOTE: now we import it from our "public" folder, but when the Authorisation lib -// will move to the "es_ui_shared" plugin, it will be imported from its "static" folder -import { Privileges } from '../../../public/app/lib/authorization'; -import { Plugins } from '../../shim'; - -let xpackMainPlugin: any; - -export function registerAppRoutes(router: Router, plugins: Plugins) { - xpackMainPlugin = plugins.xpack_main; - router.get('privileges', getPrivilegesHandler); -} - -export function getXpackMainPlugin() { - return xpackMainPlugin; -} - -const extractMissingPrivileges = (privilegesObject: { [key: string]: boolean } = {}): string[] => - Object.keys(privilegesObject).reduce((privileges: string[], privilegeName: string): string[] => { - if (!privilegesObject[privilegeName]) { - privileges.push(privilegeName); - } - return privileges; - }, []); - -export const getPrivilegesHandler: RouterRouteHandler = async ( - req, - callWithRequest -): Promise => { - const xpackInfo = getXpackMainPlugin() && getXpackMainPlugin().info; - if (!xpackInfo) { - // xpackInfo is updated via poll, so it may not be available until polling has begun. - // In this rare situation, tell the client the service is temporarily unavailable. - throw wrapCustomError(new Error('Security info unavailable'), 503); - } - - const privilegesResult: Privileges = { - hasAllPrivileges: true, - missingPrivileges: { - cluster: [], - index: [], - }, - }; - - const securityInfo = xpackInfo && xpackInfo.isAvailable() && xpackInfo.feature('security'); - if (!securityInfo || !securityInfo.isAvailable() || !securityInfo.isEnabled()) { - // If security isn't enabled, let the user use app. - return privilegesResult; - } - - // Get cluster priviliges - const { has_all_requested: hasAllPrivileges, cluster } = await callWithRequest( - 'transport.request', - { - path: '/_security/user/_has_privileges', - method: 'POST', - body: { - cluster: APP_CLUSTER_PRIVILEGES, - }, - } - ); - - // Find missing cluster privileges and set overall app privileges - privilegesResult.missingPrivileges.cluster = extractMissingPrivileges(cluster); - privilegesResult.hasAllPrivileges = hasAllPrivileges; - - // Get all index privileges the user has - const { indices } = await callWithRequest('transport.request', { - path: '/_security/user/_privileges', - method: 'GET', - }); - - // Check if they have all the required index privileges for at least one index - const oneIndexWithAllPrivileges = indices.find(({ privileges }: { privileges: string[] }) => { - if (privileges.includes('all')) { - return true; - } - - const indexHasAllPrivileges = APP_INDEX_PRIVILEGES.every(privilege => - privileges.includes(privilege) - ); - - return indexHasAllPrivileges; - }); - - // If they don't, return list of required index privileges - if (!oneIndexWithAllPrivileges) { - privilegesResult.missingPrivileges.index = [...APP_INDEX_PRIVILEGES]; - } - - return privilegesResult; -}; diff --git a/x-pack/legacy/plugins/transform/server/routes/api/register_routes.ts b/x-pack/legacy/plugins/transform/server/routes/api/register_routes.ts deleted file mode 100644 index c01647c598d86..0000000000000 --- a/x-pack/legacy/plugins/transform/server/routes/api/register_routes.ts +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -import { Router } from '../../../../../server/lib/create_router'; -import { Plugins } from '../../shim'; -import { registerAppRoutes } from './app'; -import { registerTransformsRoutes } from './transforms'; - -export const registerRoutes = (router: Router, plugins: Plugins): void => { - registerAppRoutes(router, plugins); - registerTransformsRoutes(router, plugins); -}; diff --git a/x-pack/legacy/plugins/transform/server/routes/api/transform_audit_messages.ts b/x-pack/legacy/plugins/transform/server/routes/api/transform_audit_messages.ts deleted file mode 100644 index c4b5fbd4d3b60..0000000000000 --- a/x-pack/legacy/plugins/transform/server/routes/api/transform_audit_messages.ts +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { CallCluster } from 'src/legacy/core_plugins/elasticsearch'; - -import { AuditMessage } from '../../../common/types/messages'; - -const ML_DF_NOTIFICATION_INDEX_PATTERN = '.transform-notifications-read'; -const SIZE = 500; - -interface BoolQuery { - bool: { [key: string]: any }; -} - -export function transformAuditMessagesProvider(callWithRequest: CallCluster) { - // search for audit messages, - // transformId is optional. without it, all transforms will be listed. - async function getTransformAuditMessages(transformId: string) { - const query: BoolQuery = { - bool: { - filter: [ - { - bool: { - must_not: { - term: { - level: 'activity', - }, - }, - }, - }, - ], - }, - }; - - // if no transformId specified, load all of the messages - if (transformId !== undefined) { - query.bool.filter.push({ - bool: { - should: [ - { - term: { - transform_id: '', // catch system messages - }, - }, - { - term: { - transform_id: transformId, // messages for specified transformId - }, - }, - ], - }, - }); - } - - try { - const resp = await callWithRequest('search', { - index: ML_DF_NOTIFICATION_INDEX_PATTERN, - ignore_unavailable: true, - rest_total_hits_as_int: true, - size: SIZE, - body: { - sort: [{ timestamp: { order: 'desc' } }, { transform_id: { order: 'asc' } }], - query, - }, - }); - - let messages = []; - if (resp.hits.total !== 0) { - messages = resp.hits.hits.map((hit: AuditMessage) => hit._source); - messages.reverse(); - } - return messages; - } catch (e) { - throw e; - } - } - - return { - getTransformAuditMessages, - }; -} diff --git a/x-pack/legacy/plugins/transform/server/routes/api/transforms.ts b/x-pack/legacy/plugins/transform/server/routes/api/transforms.ts deleted file mode 100644 index 6e833854a24c9..0000000000000 --- a/x-pack/legacy/plugins/transform/server/routes/api/transforms.ts +++ /dev/null @@ -1,261 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -import { CallCluster } from 'src/legacy/core_plugins/elasticsearch'; -import { Router, RouterRouteHandler } from '../../../../../server/lib/create_router'; -import { wrapEsError } from '../../../../../server/lib/create_router/error_wrappers'; -import { Plugins } from '../../shim'; -import { TRANSFORM_STATE } from '../../../public/app/common'; -import { - TransformEndpointRequest, - TransformEndpointResult, -} from '../../../public/app/hooks/use_api_types'; -import { TransformId } from '../../../public/app/common/transform'; -import { isRequestTimeout, fillResultsWithTimeouts } from './error_utils'; -import { transformAuditMessagesProvider } from './transform_audit_messages'; - -enum TRANSFORM_ACTIONS { - STOP = 'stop', - START = 'start', - DELETE = 'delete', -} - -interface StartOptions { - transformId: TransformId; -} - -interface StopOptions { - transformId: TransformId; - force: boolean; - waitForCompletion?: boolean; -} - -export function registerTransformsRoutes(router: Router, plugins: Plugins) { - router.get('transforms', getTransformHandler); - router.get('transforms/{transformId}', getTransformHandler); - router.get('transforms/_stats', getTransformStatsHandler); - router.get('transforms/{transformId}/_stats', getTransformStatsHandler); - router.get('transforms/{transformId}/messages', getTransformMessagesHandler); - router.put('transforms/{transformId}', putTransformHandler); - router.post('delete_transforms', deleteTransformsHandler); - router.post('transforms/_preview', previewTransformHandler); - router.post('start_transforms', startTransformsHandler); - router.post('stop_transforms', stopTransformsHandler); - router.post('es_search', esSearchHandler); -} - -const getTransformHandler: RouterRouteHandler = async (req, callWithRequest) => { - const { transformId } = req.params; - const options = { - ...(transformId !== undefined ? { transformId } : {}), - }; - - try { - return await callWithRequest('transform.getTransforms', options); - } catch (e) { - return { error: wrapEsError(e) }; - } -}; - -const getTransformStatsHandler: RouterRouteHandler = async (req, callWithRequest) => { - const { transformId } = req.params; - const options = { - ...(transformId !== undefined ? { transformId } : {}), - }; - - try { - return await callWithRequest('transform.getTransformsStats', options); - } catch (e) { - return { error: wrapEsError(e) }; - } -}; - -const deleteTransformsHandler: RouterRouteHandler = async (req, callWithRequest) => { - const transformsInfo = req.payload as TransformEndpointRequest[]; - - try { - return await deleteTransforms(transformsInfo, callWithRequest); - } catch (e) { - return { error: wrapEsError(e) }; - } -}; - -const putTransformHandler: RouterRouteHandler = async (req, callWithRequest) => { - const { transformId } = req.params; - - const response: { - transformsCreated: Array<{ transform: string }>; - errors: any[]; - } = { - transformsCreated: [], - errors: [], - }; - - await callWithRequest('transform.createTransform', { body: req.payload, transformId }) - .then(() => response.transformsCreated.push({ transform: transformId })) - .catch(e => - response.errors.push({ - id: transformId, - error: wrapEsError(e), - }) - ); - - return response; -}; - -async function deleteTransforms( - transformsInfo: TransformEndpointRequest[], - callWithRequest: CallCluster -) { - const results: TransformEndpointResult = {}; - - for (const transformInfo of transformsInfo) { - const transformId = transformInfo.id; - try { - if (transformInfo.state === TRANSFORM_STATE.FAILED) { - try { - await callWithRequest('transform.stopTransform', { - transformId, - force: true, - waitForCompletion: true, - } as StopOptions); - } catch (e) { - if (isRequestTimeout(e)) { - return fillResultsWithTimeouts({ - results, - id: transformId, - items: transformsInfo, - action: TRANSFORM_ACTIONS.DELETE, - }); - } - } - } - - await callWithRequest('transform.deleteTransform', { transformId }); - results[transformId] = { success: true }; - } catch (e) { - if (isRequestTimeout(e)) { - return fillResultsWithTimeouts({ - results, - id: transformInfo.id, - items: transformsInfo, - action: TRANSFORM_ACTIONS.DELETE, - }); - } - results[transformId] = { success: false, error: JSON.stringify(e) }; - } - } - return results; -} - -const previewTransformHandler: RouterRouteHandler = async (req, callWithRequest) => { - try { - return await callWithRequest('transform.getTransformsPreview', { body: req.payload }); - } catch (e) { - return wrapEsError(e); - } -}; - -const startTransformsHandler: RouterRouteHandler = async (req, callWithRequest) => { - const { transformsInfo } = req.payload as { - transformsInfo: TransformEndpointRequest[]; - }; - - try { - return await startTransforms(transformsInfo, callWithRequest); - } catch (e) { - return wrapEsError(e); - } -}; - -async function startTransforms( - transformsInfo: TransformEndpointRequest[], - callWithRequest: CallCluster -) { - const results: TransformEndpointResult = {}; - - for (const transformInfo of transformsInfo) { - const transformId = transformInfo.id; - try { - await callWithRequest('transform.startTransform', { transformId } as StartOptions); - results[transformId] = { success: true }; - } catch (e) { - if (isRequestTimeout(e)) { - return fillResultsWithTimeouts({ - results, - id: transformId, - items: transformsInfo, - action: TRANSFORM_ACTIONS.START, - }); - } - results[transformId] = { success: false, error: JSON.stringify(e) }; - } - } - return results; -} - -const stopTransformsHandler: RouterRouteHandler = async (req, callWithRequest) => { - const { transformsInfo } = req.payload as { - transformsInfo: TransformEndpointRequest[]; - }; - - try { - return await stopTransforms(transformsInfo, callWithRequest); - } catch (e) { - return wrapEsError(e); - } -}; - -async function stopTransforms( - transformsInfo: TransformEndpointRequest[], - callWithRequest: CallCluster -) { - const results: TransformEndpointResult = {}; - - for (const transformInfo of transformsInfo) { - const transformId = transformInfo.id; - try { - await callWithRequest('transform.stopTransform', { - transformId, - force: - transformInfo.state !== undefined - ? transformInfo.state === TRANSFORM_STATE.FAILED - : false, - waitForCompletion: true, - } as StopOptions); - results[transformId] = { success: true }; - } catch (e) { - if (isRequestTimeout(e)) { - return fillResultsWithTimeouts({ - results, - id: transformId, - items: transformsInfo, - action: TRANSFORM_ACTIONS.STOP, - }); - } - results[transformId] = { success: false, error: JSON.stringify(e) }; - } - } - return results; -} - -const getTransformMessagesHandler: RouterRouteHandler = async (req, callWithRequest) => { - const { getTransformAuditMessages } = transformAuditMessagesProvider(callWithRequest); - const { transformId } = req.params; - - try { - return await getTransformAuditMessages(transformId); - } catch (e) { - return wrapEsError(e); - } -}; - -const esSearchHandler: RouterRouteHandler = async (req, callWithRequest) => { - try { - return await callWithRequest('search', req.payload); - } catch (e) { - return { error: wrapEsError(e) }; - } -}; diff --git a/x-pack/legacy/plugins/transform/server/shim.ts b/x-pack/legacy/plugins/transform/server/shim.ts deleted file mode 100644 index 8f477d86441f4..0000000000000 --- a/x-pack/legacy/plugins/transform/server/shim.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { Legacy } from 'kibana'; -import { createRouter, Router } from '../../../server/lib/create_router'; -import { registerLicenseChecker } from '../../../server/lib/register_license_checker'; -import { elasticsearchJsPlugin } from './client/elasticsearch_transform'; -export interface Core { - http: { - createRouter(basePath: string): Router; - }; -} - -export interface Plugins { - license: { - registerLicenseChecker: typeof registerLicenseChecker; - }; - xpack_main: any; - elasticsearch: any; -} - -export function createServerShim( - server: Legacy.Server, - pluginId: string -): { core: Core; plugins: Plugins } { - return { - core: { - http: { - createRouter: (basePath: string) => - createRouter(server, pluginId, basePath, { - plugins: [elasticsearchJsPlugin], - }), - }, - }, - plugins: { - license: { - registerLicenseChecker, - }, - xpack_main: server.plugins.xpack_main, - elasticsearch: server.plugins.elasticsearch, - }, - }; -} diff --git a/x-pack/plugins/transform/kibana.json b/x-pack/plugins/transform/kibana.json new file mode 100644 index 0000000000000..87e38f83ef640 --- /dev/null +++ b/x-pack/plugins/transform/kibana.json @@ -0,0 +1,15 @@ +{ + "id": "transform", + "version": "kibana", + "server": true, + "ui": false, + "requiredPlugins": [ + "home", + "licensing", + "management" + ], + "optionalPlugins": [ + "usageCollection" + ], + "configPath": ["xpack", "transform"] +} diff --git a/x-pack/legacy/plugins/transform/server/client/elasticsearch_transform.ts b/x-pack/plugins/transform/server/client/elasticsearch_transform.ts similarity index 100% rename from x-pack/legacy/plugins/transform/server/client/elasticsearch_transform.ts rename to x-pack/plugins/transform/server/client/elasticsearch_transform.ts diff --git a/x-pack/plugins/transform/server/index.ts b/x-pack/plugins/transform/server/index.ts new file mode 100644 index 0000000000000..7b7cf3ee44fb5 --- /dev/null +++ b/x-pack/plugins/transform/server/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { PluginInitializerContext } from 'src/core/server'; + +import { TransformServerPlugin } from './plugin'; + +export const plugin = (ctx: PluginInitializerContext) => new TransformServerPlugin(ctx); diff --git a/x-pack/plugins/transform/server/plugin.ts b/x-pack/plugins/transform/server/plugin.ts new file mode 100644 index 0000000000000..7da991bc02b37 --- /dev/null +++ b/x-pack/plugins/transform/server/plugin.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { i18n } from '@kbn/i18n'; +import { + CoreSetup, + Plugin, + IScopedClusterClient, + Logger, + PluginInitializerContext, +} from 'src/core/server'; + +import { LicenseType } from '../../licensing/common/types'; + +import { elasticsearchJsPlugin } from './client/elasticsearch_transform'; +import { Dependencies } from './types'; +import { ApiRoutes } from './routes'; +import { License } from './services'; + +declare module 'kibana/server' { + interface RequestHandlerContext { + transform?: { + dataClient: IScopedClusterClient; + }; + } +} + +const basicLicense: LicenseType = 'basic'; + +const PLUGIN = { + id: 'transform', + minimumLicenseType: basicLicense, + getI18nName: (): string => + i18n.translate('xpack.transform.appTitle', { + defaultMessage: 'Transforms', + }), +}; + +export class TransformServerPlugin implements Plugin<{}, void, any, any> { + private readonly apiRoutes: ApiRoutes; + private readonly license: License; + private readonly logger: Logger; + + constructor(initContext: PluginInitializerContext) { + this.logger = initContext.logger.get(); + this.apiRoutes = new ApiRoutes(); + this.license = new License(); + } + + setup({ elasticsearch, http }: CoreSetup, { licensing }: Dependencies): {} { + const router = http.createRouter(); + + this.license.setup( + { + pluginId: PLUGIN.id, + minimumLicenseType: PLUGIN.minimumLicenseType, + defaultErrorMessage: i18n.translate('xpack.transform.licenseCheckErrorMessage', { + defaultMessage: 'License check failed', + }), + }, + { + licensing, + logger: this.logger, + } + ); + + this.apiRoutes.setup({ + router, + license: this.license, + }); + + // Can access via new platform router's handler function 'context' parameter - context.transform.client + const transformClient = elasticsearch.createClient('transform', { + plugins: [elasticsearchJsPlugin], + }); + http.registerRouteHandlerContext('transform', (context, request) => { + return { + dataClient: transformClient.asScoped(request), + }; + }); + + return {}; + } + + start() {} + stop() {} +} diff --git a/x-pack/legacy/plugins/transform/server/routes/api/error_utils.ts b/x-pack/plugins/transform/server/routes/api/error_utils.ts similarity index 80% rename from x-pack/legacy/plugins/transform/server/routes/api/error_utils.ts rename to x-pack/plugins/transform/server/routes/api/error_utils.ts index 094c0308ff20f..d09152bf1a603 100644 --- a/x-pack/legacy/plugins/transform/server/routes/api/error_utils.ts +++ b/x-pack/plugins/transform/server/routes/api/error_utils.ts @@ -4,11 +4,16 @@ * you may not use this file except in compliance with the Elastic License. */ +import { boomify, isBoom } from 'boom'; + import { i18n } from '@kbn/i18n'; + +import { ResponseError, CustomHttpResponseOptions } from 'src/core/server'; + import { TransformEndpointRequest, TransformEndpointResult, -} from '../../../public/app/hooks/use_api_types'; +} from '../../../../../legacy/plugins/transform/public/app/hooks/use_api_types'; const REQUEST_TIMEOUT = 'RequestTimeout'; @@ -71,3 +76,12 @@ export function fillResultsWithTimeouts({ results, id, items, action }: Params) return accumResults; }, newResults); } + +export function wrapError(error: any): CustomHttpResponseOptions { + const boom = isBoom(error) ? error : boomify(error, { statusCode: error.status }); + return { + body: boom, + headers: boom.output.headers, + statusCode: boom.output.statusCode, + }; +} diff --git a/x-pack/plugins/transform/server/routes/api/privileges.ts b/x-pack/plugins/transform/server/routes/api/privileges.ts new file mode 100644 index 0000000000000..6003a88ffa40c --- /dev/null +++ b/x-pack/plugins/transform/server/routes/api/privileges.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { + APP_CLUSTER_PRIVILEGES, + APP_INDEX_PRIVILEGES, +} from '../../../../../legacy/plugins/transform/common/constants'; +// NOTE: now we import it from our "public" folder, but when the Authorisation lib +// will move to the "es_ui_shared" plugin, it will be imported from its "static" folder +import { Privileges } from '../../../../../legacy/plugins/transform/public/app/lib/authorization'; + +import { RouteDependencies } from '../../types'; +import { addBasePath } from '../index'; + +export function registerPrivilegesRoute({ router, license }: RouteDependencies) { + router.get( + { path: addBasePath('privileges'), validate: {} }, + license.guardApiRoute(async (ctx, req, res) => { + const privilegesResult: Privileges = { + hasAllPrivileges: true, + missingPrivileges: { + cluster: [], + index: [], + }, + }; + + if (license.getStatus().isSecurityEnabled === false) { + // If security isn't enabled, let the user use app. + return res.ok({ body: privilegesResult }); + } + + // Get cluster priviliges + const { + has_all_requested: hasAllPrivileges, + cluster, + } = await ctx.transform!.dataClient.callAsCurrentUser('transport.request', { + path: '/_security/user/_has_privileges', + method: 'POST', + body: { + cluster: APP_CLUSTER_PRIVILEGES, + }, + }); + + // Find missing cluster privileges and set overall app privileges + privilegesResult.missingPrivileges.cluster = extractMissingPrivileges(cluster); + privilegesResult.hasAllPrivileges = hasAllPrivileges; + + // Get all index privileges the user has + const { indices } = await ctx.transform!.dataClient.callAsCurrentUser('transport.request', { + path: '/_security/user/_privileges', + method: 'GET', + }); + + // Check if they have all the required index privileges for at least one index + const oneIndexWithAllPrivileges = indices.find(({ privileges }: { privileges: string[] }) => { + if (privileges.includes('all')) { + return true; + } + + const indexHasAllPrivileges = APP_INDEX_PRIVILEGES.every(privilege => + privileges.includes(privilege) + ); + + return indexHasAllPrivileges; + }); + + // If they don't, return list of required index privileges + if (!oneIndexWithAllPrivileges) { + privilegesResult.missingPrivileges.index = [...APP_INDEX_PRIVILEGES]; + } + + return res.ok({ body: privilegesResult }); + }) + ); +} + +const extractMissingPrivileges = (privilegesObject: { [key: string]: boolean } = {}): string[] => + Object.keys(privilegesObject).reduce((privileges: string[], privilegeName: string): string[] => { + if (!privilegesObject[privilegeName]) { + privileges.push(privilegeName); + } + return privileges; + }, []); diff --git a/x-pack/plugins/transform/server/routes/api/schema.ts b/x-pack/plugins/transform/server/routes/api/schema.ts new file mode 100644 index 0000000000000..0b994406d324d --- /dev/null +++ b/x-pack/plugins/transform/server/routes/api/schema.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { schema } from '@kbn/config-schema'; + +export const schemaTransformId = { + params: schema.object({ + transformId: schema.string(), + }), +}; + +export interface SchemaTransformId { + transformId: string; +} diff --git a/x-pack/plugins/transform/server/routes/api/transforms.ts b/x-pack/plugins/transform/server/routes/api/transforms.ts new file mode 100644 index 0000000000000..7aaae1f1c7039 --- /dev/null +++ b/x-pack/plugins/transform/server/routes/api/transforms.ts @@ -0,0 +1,360 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { schema } from '@kbn/config-schema'; + +import { RequestHandler } from 'kibana/server'; +import { CallCluster } from 'src/legacy/core_plugins/elasticsearch'; +import { wrapEsError } from '../../../../../legacy/server/lib/create_router/error_wrappers'; + +import { TRANSFORM_STATE } from '../../../../../legacy/plugins/transform/public/app/common'; +import { + TransformEndpointRequest, + TransformEndpointResult, +} from '../../../../../legacy/plugins/transform/public/app/hooks/use_api_types'; +import { TransformId } from '../../../../../legacy/plugins/transform/public/app/common/transform'; + +import { RouteDependencies } from '../../types'; + +import { addBasePath } from '../index'; + +import { isRequestTimeout, fillResultsWithTimeouts, wrapError } from './error_utils'; +import { schemaTransformId, SchemaTransformId } from './schema'; +import { registerTransformsAuditMessagesRoutes } from './transforms_audit_messages'; + +enum TRANSFORM_ACTIONS { + STOP = 'stop', + START = 'start', + DELETE = 'delete', +} + +interface StopOptions { + transformId: TransformId; + force: boolean; + waitForCompletion?: boolean; +} + +export function registerTransformsRoutes(routeDependencies: RouteDependencies) { + const { router, license } = routeDependencies; + router.get( + { path: addBasePath('transforms'), validate: false }, + license.guardApiRoute(async (ctx, req, res) => { + const options = {}; + try { + const transforms = await getTransforms( + options, + ctx.transform!.dataClient.callAsCurrentUser + ); + return res.ok({ body: transforms }); + } catch (e) { + return res.customError(wrapError(wrapEsError(e))); + } + }) + ); + router.get( + { + path: addBasePath('transforms/{transformId}'), + validate: schemaTransformId, + }, + license.guardApiRoute(async (ctx, req, res) => { + const { transformId } = req.params as SchemaTransformId; + const options = { + ...(transformId !== undefined ? { transformId } : {}), + }; + try { + const transforms = await getTransforms( + options, + ctx.transform!.dataClient.callAsCurrentUser + ); + return res.ok({ body: transforms }); + } catch (e) { + return res.customError(wrapError(wrapEsError(e))); + } + }) + ); + router.get( + { path: addBasePath('transforms/_stats'), validate: false }, + license.guardApiRoute(async (ctx, req, res) => { + const options = {}; + try { + const stats = await ctx.transform!.dataClient.callAsCurrentUser( + 'transform.getTransformsStats', + options + ); + return res.ok({ body: stats }); + } catch (e) { + return res.customError(wrapError(wrapEsError(e))); + } + }) + ); + router.get( + { + path: addBasePath('transforms/{transformId}/_stats'), + validate: schemaTransformId, + }, + license.guardApiRoute(async (ctx, req, res) => { + const { transformId } = req.params as SchemaTransformId; + const options = { + ...(transformId !== undefined ? { transformId } : {}), + }; + try { + const stats = await ctx.transform!.dataClient.callAsCurrentUser( + 'transform.getTransformsStats', + options + ); + return res.ok({ body: stats }); + } catch (e) { + return res.customError(wrapError(wrapEsError(e))); + } + }) + ); + registerTransformsAuditMessagesRoutes(routeDependencies); + router.put( + { + path: addBasePath('transforms/{transformId}'), + validate: { + ...schemaTransformId, + body: schema.maybe(schema.any()), + }, + }, + license.guardApiRoute(async (ctx, req, res) => { + const { transformId } = req.params as SchemaTransformId; + + const response: { + transformsCreated: Array<{ transform: string }>; + errors: any[]; + } = { + transformsCreated: [], + errors: [], + }; + + await ctx + .transform!.dataClient.callAsCurrentUser('transform.createTransform', { + body: req.body, + transformId, + }) + .then(() => response.transformsCreated.push({ transform: transformId })) + .catch(e => + response.errors.push({ + id: transformId, + error: wrapEsError(e), + }) + ); + + return res.ok({ body: response }); + }) + ); + router.post( + { + path: addBasePath('delete_transforms'), + validate: { + body: schema.maybe(schema.any()), + }, + }, + license.guardApiRoute(async (ctx, req, res) => { + const transformsInfo = req.body as TransformEndpointRequest[]; + + try { + return res.ok({ + body: await deleteTransforms(transformsInfo, ctx.transform!.dataClient.callAsCurrentUser), + }); + } catch (e) { + return res.customError(wrapError(wrapEsError(e))); + } + }) + ); + router.post( + { + path: addBasePath('transforms/_preview'), + validate: { + body: schema.maybe(schema.any()), + }, + }, + license.guardApiRoute(previewTransformHandler) + ); + router.post( + { + path: addBasePath('start_transforms'), + validate: { + body: schema.maybe(schema.any()), + }, + }, + license.guardApiRoute(startTransformsHandler) + ); + router.post( + { + path: addBasePath('stop_transforms'), + validate: { + body: schema.maybe(schema.any()), + }, + }, + license.guardApiRoute(stopTransformsHandler) + ); + router.post( + { + path: addBasePath('es_search'), + validate: { + body: schema.maybe(schema.any()), + }, + }, + license.guardApiRoute(async (ctx, req, res) => { + try { + return res.ok({ + body: await ctx.transform!.dataClient.callAsCurrentUser('search', req.body), + }); + } catch (e) { + return res.customError(wrapError(wrapEsError(e))); + } + }) + ); +} + +const getTransforms = async (options: { transformId?: string }, callAsCurrentUser: CallCluster) => { + return await callAsCurrentUser('transform.getTransforms', options); +}; + +async function deleteTransforms( + transformsInfo: TransformEndpointRequest[], + callAsCurrentUser: CallCluster +) { + const results: TransformEndpointResult = {}; + + for (const transformInfo of transformsInfo) { + const transformId = transformInfo.id; + try { + if (transformInfo.state === TRANSFORM_STATE.FAILED) { + try { + await callAsCurrentUser('transform.stopTransform', { + transformId, + force: true, + waitForCompletion: true, + } as StopOptions); + } catch (e) { + if (isRequestTimeout(e)) { + return fillResultsWithTimeouts({ + results, + id: transformId, + items: transformsInfo, + action: TRANSFORM_ACTIONS.DELETE, + }); + } + } + } + + await callAsCurrentUser('transform.deleteTransform', { transformId }); + results[transformId] = { success: true }; + } catch (e) { + if (isRequestTimeout(e)) { + return fillResultsWithTimeouts({ + results, + id: transformInfo.id, + items: transformsInfo, + action: TRANSFORM_ACTIONS.DELETE, + }); + } + results[transformId] = { success: false, error: JSON.stringify(e) }; + } + } + return results; +} + +const previewTransformHandler: RequestHandler = async (ctx, req, res) => { + try { + return res.ok({ + body: await ctx.transform!.dataClient.callAsCurrentUser('transform.getTransformsPreview', { + body: req.body, + }), + }); + } catch (e) { + return res.customError(wrapError(wrapEsError(e))); + } +}; + +const startTransformsHandler: RequestHandler = async (ctx, req, res) => { + const { transformsInfo } = req.body as { + transformsInfo: TransformEndpointRequest[]; + }; + + try { + return res.ok({ + body: await startTransforms(transformsInfo, ctx.transform!.dataClient.callAsCurrentUser), + }); + } catch (e) { + return res.customError(wrapError(wrapEsError(e))); + } +}; + +async function startTransforms( + transformsInfo: TransformEndpointRequest[], + callAsCurrentUser: CallCluster +) { + const results: TransformEndpointResult = {}; + + for (const transformInfo of transformsInfo) { + const transformId = transformInfo.id; + try { + await callAsCurrentUser('transform.startTransform', { transformId } as SchemaTransformId); + results[transformId] = { success: true }; + } catch (e) { + if (isRequestTimeout(e)) { + return fillResultsWithTimeouts({ + results, + id: transformId, + items: transformsInfo, + action: TRANSFORM_ACTIONS.START, + }); + } + results[transformId] = { success: false, error: JSON.stringify(e) }; + } + } + return results; +} + +const stopTransformsHandler: RequestHandler = async (ctx, req, res) => { + const { transformsInfo } = req.body as { + transformsInfo: TransformEndpointRequest[]; + }; + + try { + return res.ok({ + body: await stopTransforms(transformsInfo, ctx.transform!.dataClient.callAsCurrentUser), + }); + } catch (e) { + return res.customError(wrapError(wrapEsError(e))); + } +}; + +async function stopTransforms( + transformsInfo: TransformEndpointRequest[], + callAsCurrentUser: CallCluster +) { + const results: TransformEndpointResult = {}; + + for (const transformInfo of transformsInfo) { + const transformId = transformInfo.id; + try { + await callAsCurrentUser('transform.stopTransform', { + transformId, + force: + transformInfo.state !== undefined + ? transformInfo.state === TRANSFORM_STATE.FAILED + : false, + waitForCompletion: true, + } as StopOptions); + results[transformId] = { success: true }; + } catch (e) { + if (isRequestTimeout(e)) { + return fillResultsWithTimeouts({ + results, + id: transformId, + items: transformsInfo, + action: TRANSFORM_ACTIONS.STOP, + }); + } + results[transformId] = { success: false, error: JSON.stringify(e) }; + } + } + return results; +} diff --git a/x-pack/plugins/transform/server/routes/api/transforms_audit_messages.ts b/x-pack/plugins/transform/server/routes/api/transforms_audit_messages.ts new file mode 100644 index 0000000000000..422fdec7ab77e --- /dev/null +++ b/x-pack/plugins/transform/server/routes/api/transforms_audit_messages.ts @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { AuditMessage } from '../../../../../legacy/plugins/transform/common/types/messages'; +import { wrapEsError } from '../../../../../legacy/server/lib/create_router/error_wrappers'; + +import { RouteDependencies } from '../../types'; + +import { addBasePath } from '../index'; + +import { wrapError } from './error_utils'; +import { schemaTransformId, SchemaTransformId } from './schema'; + +const ML_DF_NOTIFICATION_INDEX_PATTERN = '.transform-notifications-read'; +const SIZE = 500; + +interface BoolQuery { + bool: { [key: string]: any }; +} + +export function registerTransformsAuditMessagesRoutes({ router, license }: RouteDependencies) { + router.get( + { path: addBasePath('transforms/{transformId}/messages'), validate: schemaTransformId }, + license.guardApiRoute(async (ctx, req, res) => { + const { transformId } = req.params as SchemaTransformId; + + // search for audit messages, + // transformId is optional. without it, all transforms will be listed. + const query: BoolQuery = { + bool: { + filter: [ + { + bool: { + must_not: { + term: { + level: 'activity', + }, + }, + }, + }, + ], + }, + }; + + // if no transformId specified, load all of the messages + if (transformId !== undefined) { + query.bool.filter.push({ + bool: { + should: [ + { + term: { + transform_id: '', // catch system messages + }, + }, + { + term: { + transform_id: transformId, // messages for specified transformId + }, + }, + ], + }, + }); + } + + try { + const resp = await ctx.transform!.dataClient.callAsCurrentUser('search', { + index: ML_DF_NOTIFICATION_INDEX_PATTERN, + ignore_unavailable: true, + rest_total_hits_as_int: true, + size: SIZE, + body: { + sort: [{ timestamp: { order: 'desc' } }, { transform_id: { order: 'asc' } }], + query, + }, + }); + + let messages = []; + if (resp.hits.total !== 0) { + messages = resp.hits.hits.map((hit: AuditMessage) => hit._source); + messages.reverse(); + } + return res.ok({ body: messages }); + } catch (e) { + return res.customError(wrapError(wrapEsError(e))); + } + }) + ); +} diff --git a/x-pack/plugins/transform/server/routes/index.ts b/x-pack/plugins/transform/server/routes/index.ts new file mode 100644 index 0000000000000..953490920cbcb --- /dev/null +++ b/x-pack/plugins/transform/server/routes/index.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { RouteDependencies } from '../types'; + +import { registerPrivilegesRoute } from './api/privileges'; +import { registerTransformsRoutes } from './api/transforms'; + +import { API_BASE_PATH } from '../../../../legacy/plugins/transform/common/constants'; + +export const addBasePath = (uri: string): string => `${API_BASE_PATH}${uri}`; + +export class ApiRoutes { + setup(dependencies: RouteDependencies) { + registerPrivilegesRoute(dependencies); + registerTransformsRoutes(dependencies); + } + + start() {} + stop() {} +} diff --git a/x-pack/plugins/transform/server/services/index.ts b/x-pack/plugins/transform/server/services/index.ts new file mode 100644 index 0000000000000..b7a45e59549eb --- /dev/null +++ b/x-pack/plugins/transform/server/services/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { License } from './license'; diff --git a/x-pack/plugins/transform/server/services/license.ts b/x-pack/plugins/transform/server/services/license.ts new file mode 100644 index 0000000000000..93346160c6f44 --- /dev/null +++ b/x-pack/plugins/transform/server/services/license.ts @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { Logger } from 'src/core/server'; +import { + IKibanaResponse, + KibanaRequest, + KibanaResponseFactory, + RequestHandler, + RequestHandlerContext, +} from 'kibana/server'; + +import { LicensingPluginSetup, LicenseType, LICENSE_CHECK_STATE } from '../../../licensing/server'; + +export interface LicenseStatus { + isValid: boolean; + isSecurityEnabled: boolean; + message?: string; +} + +interface SetupSettings { + pluginId: string; + minimumLicenseType: LicenseType; + defaultErrorMessage: string; +} + +export class License { + private licenseStatus: LicenseStatus = { + isValid: false, + isSecurityEnabled: false, + message: 'Invalid License', + }; + + setup( + { pluginId, minimumLicenseType, defaultErrorMessage }: SetupSettings, + { licensing, logger }: { licensing: LicensingPluginSetup; logger: Logger } + ) { + licensing.license$.subscribe(license => { + const { state, message } = license.check(pluginId, minimumLicenseType); + const hasRequiredLicense = state === LICENSE_CHECK_STATE.Valid; + + const securityFeature = license.getFeature('security'); + const isSecurityEnabled = + securityFeature !== undefined && + securityFeature.isAvailable === true && + securityFeature.isEnabled === true; + + if (hasRequiredLicense) { + this.licenseStatus = { isValid: true, isSecurityEnabled }; + } else { + this.licenseStatus = { + isValid: false, + isSecurityEnabled, + message: message || defaultErrorMessage, + }; + if (message) { + logger.info(message); + } + } + }); + } + + guardApiRoute(handler: RequestHandler) { + const license = this; + + return function licenseCheck( + ctx: RequestHandlerContext, + request: KibanaRequest, + response: KibanaResponseFactory + ): IKibanaResponse | Promise> { + const licenseStatus = license.getStatus(); + + if (!licenseStatus.isValid) { + return response.customError({ + body: { + message: licenseStatus.message || '', + }, + statusCode: 403, + }); + } + + return handler(ctx, request, response); + }; + } + + getStatus() { + return this.licenseStatus; + } +} diff --git a/x-pack/plugins/transform/server/types.ts b/x-pack/plugins/transform/server/types.ts new file mode 100644 index 0000000000000..5fcc23a6d9f48 --- /dev/null +++ b/x-pack/plugins/transform/server/types.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IRouter } from 'src/core/server'; +import { LicensingPluginSetup } from '../../licensing/server'; +import { License } from './services'; + +export interface Dependencies { + licensing: LicensingPluginSetup; +} + +export interface RouteDependencies { + router: IRouter; + license: License; +}