From 8e0f4e3bcec315678e19ab526164baf8eb1a6168 Mon Sep 17 00:00:00 2001 From: Vignesh Shanmugam Date: Mon, 5 Dec 2022 09:00:30 -0800 Subject: [PATCH] feat: use new kibana api for pushing monitors (#649) * feat: use new kibana api for pushing monitors Co-authored-by: Andrew Cholakian Co-authored-by: shahzad31 Co-authored-by: Dominique Clarke --- .../push/__snapshots__/index.test.ts.snap | 18 -- .../push/__snapshots__/request.test.ts.snap | 2 +- __tests__/push/index.test.ts | 182 +++++----------- __tests__/push/monitor.test.ts | 71 +++---- __tests__/push/request.test.ts | 2 +- __tests__/utils/kibana-test-server.ts | 87 ++++++++ package-lock.json | 13 +- package.json | 1 + src/core/runner.ts | 2 - src/dsl/monitor.ts | 6 + src/helpers.ts | 27 +-- src/locations/index.ts | 25 +-- src/push/index.ts | 196 ++++++++++-------- src/push/kibana_api.ts | 180 ++++++++++++++++ src/push/monitor.ts | 107 ++++++---- src/push/request.ts | 41 +++- src/push/utils.ts | 77 +++++++ 17 files changed, 688 insertions(+), 349 deletions(-) create mode 100644 __tests__/utils/kibana-test-server.ts create mode 100644 src/push/kibana_api.ts create mode 100644 src/push/utils.ts diff --git a/__tests__/push/__snapshots__/index.test.ts.snap b/__tests__/push/__snapshots__/index.test.ts.snap index 727591ac..25c15126 100644 --- a/__tests__/push/__snapshots__/index.test.ts.snap +++ b/__tests__/push/__snapshots__/index.test.ts.snap @@ -1,23 +1,5 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`Push API handle streamed response 1`] = ` -"> preparing all monitors -> creating all monitors -> 2 monitors created successfully -> journey 2 monitor updated successfully -> deleting all stale monitors -✓ Pushed: http://localhost:54455/stream/app/uptime/manage-monitors/all -" -`; - -exports[`Push API handle sync response 1`] = ` -"> preparing all monitors -> creating all monitors -> deleting all stale monitors -✓ Pushed: http://localhost:54455/sync/app/uptime/manage-monitors/all -" -`; - exports[`Push error on empty project id 1`] = ` "Aborted. Invalid synthetics project settings. diff --git a/__tests__/push/__snapshots__/request.test.ts.snap b/__tests__/push/__snapshots__/request.test.ts.snap index 0064451d..d21aa3d3 100644 --- a/__tests__/push/__snapshots__/request.test.ts.snap +++ b/__tests__/push/__snapshots__/request.test.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`Push api request format 404 error 1`] = `"✖ Please check your kibana url and try again - 404:Not Found"`; +exports[`Push api request format 404 error 1`] = `"✖ Please check your kibana url: http://foo and try again - 404:Not Found"`; exports[`Push api request format api error 1`] = ` "✖ Error diff --git a/__tests__/push/index.test.ts b/__tests__/push/index.test.ts index 85787b6a..17f400d7 100644 --- a/__tests__/push/index.test.ts +++ b/__tests__/push/index.test.ts @@ -28,7 +28,7 @@ import { mkdir, rm, writeFile } from 'fs/promises'; import { join } from 'path'; import { Monitor } from '../../src/dsl/monitor'; import { formatDuplicateError } from '../../src/push'; -import { APISchema } from '../../src/push/monitor'; +import { createKibanaTestServer } from '../utils/kibana-test-server'; import { Server } from '../utils/server'; import { CLIMock } from '../utils/test-config'; @@ -69,7 +69,7 @@ describe('Push', () => { it('error when project is not setup', async () => { const output = await runPush(); expect(output).toContain( - 'Aborted (missing synthetics config file), Project not set up corrrectly.' + 'Aborted (missing synthetics config file), Project not set up correctly.' ); }); @@ -130,27 +130,6 @@ journey('journey 1', () => monitor.use({ id: 'j1', schedule: 8 }));` await rm(testJourney, { force: true }); }); - it('push with different id when overriden', async () => { - await fakeProjectSetup( - { id: 'test-project', space: 'dummy', url: 'http://localhost:8080' }, - { locations: ['test-loc'], schedule: 3 } - ); - const testJourney = join(PROJECT_DIR, 'test.journey.ts'); - await writeFile( - testJourney, - `import {journey, monitor} from '../../../src'; -journey('journey 1', () => monitor.use({ id: 'j1' }));` - ); - const output = await runPush( - [...DEFAULT_ARGS, '-y', '--id', 'new-project'], - { - TEST_OVERRIDE: true, - } - ); - expect(output).toContain('preparing all monitors'); - await rm(testJourney, { force: true }); - }); - it('errors on duplicate browser monitors', async () => { await fakeProjectSetup( { id: 'test-project' }, @@ -214,110 +193,59 @@ heartbeat.monitors: expect(formatDuplicateError(duplicates as Set)).toMatchSnapshot(); }); - it('abort when delete is skipped', async () => { - await fakeProjectSetup( - { id: 'test-project' }, - { locations: ['test-loc'], schedule: 3 } - ); - const output = await runPush([...DEFAULT_ARGS, '-y'], { - TEST_OVERRIDE: false, - }); - expect(output).toContain('Push command Aborted'); - }); - - it('delete entire project with --yes flag', async () => { - await fakeProjectSetup( - { id: 'test-project', space: 'dummy', url: 'http://localhost:8080' }, - { locations: ['test-loc'], schedule: 3 } - ); - const output = await runPush([...DEFAULT_ARGS, '-y']); - expect(output).toContain('deleting all stale monitors'); - }); - - it('delete entire project with overrides', async () => { - await fakeProjectSetup( - { id: 'test-project', space: 'dummy', url: 'http://localhost:8080' }, - { locations: ['test-loc'], schedule: 3 } - ); - const output = await runPush([...DEFAULT_ARGS, '-y'], { - TEST_OVERRIDE: true, - }); - expect(output).toContain('deleting all stale monitors'); - }); - - describe('API', () => { - let server: Server; - beforeAll(async () => { - server = await Server.create({ port: 54455 }); - const apiRes = { failedMonitors: [], failedStaleMonitors: [] }; - server.route( - '/sync/s/dummy/api/synthetics/service/project/monitors', - (req, res) => { - res.end(JSON.stringify(apiRes)); - } - ); - server.route( - '/stream/s/dummy/api/synthetics/service/project/monitors', - async (req, res) => { - await new Promise(r => setTimeout(r, 20)); - req.on('data', chunks => { - const schema = JSON.parse(chunks.toString()) as APISchema; - res.write( - JSON.stringify( - schema.monitors.length + ' monitors created successfully' - ) + '\n' - ); - res.write( - JSON.stringify( - `${schema.monitors[1].name} monitor updated successfully` - ) + '\n' - ); - if (!schema.keep_stale) { - // write more than the stream buffer to check the broken NDJSON data - res.write( - JSON.stringify(Buffer.from('a'.repeat(70000)).toString()) + '\n' - ); - } - }); - req.on('end', () => { - res.end(JSON.stringify(apiRes)); - }); - } - ); - await fakeProjectSetup( - { - id: 'test-project', - space: 'dummy', - }, - { locations: ['test-loc'], schedule: 3 } - ); - - await writeFile( - join(PROJECT_DIR, 'test.journey.ts'), - `import {journey, monitor} from '../../../src/index'; -journey('journey 1', () => monitor.use({ id: 'j1' })); -journey('journey 2', () => monitor.use({ id: 'j2' }));` - ); - }); - afterAll(async () => { - await server.close(); - }); - - it('handle sync response', async () => { - const output = await runPush([ - '--url', - server.PREFIX + '/sync', - ...DEFAULT_ARGS, - ]); - expect(output).toMatchSnapshot(); - }); - it('handle streamed response', async () => { - const output = await runPush([ - '--url', - server.PREFIX + '/stream', - ...DEFAULT_ARGS, - ]); - expect(output).toMatchSnapshot(); + ['8.5.0', '8.6.0'].forEach(version => { + describe('API: ' + version, () => { + let server: Server; + const deleteProgress = + '8.5.0' === version + ? 'deleting all stale monitors' + : 'deleting 2 monitors'; + beforeAll(async () => { + server = await createKibanaTestServer(version); + await fakeProjectSetup( + { id: 'test-project', space: 'dummy', url: server.PREFIX }, + { locations: ['test-loc'], schedule: 3 } + ); + }); + afterAll(async () => { + await server.close(); + }); + + it('abort when delete is skipped', async () => { + const output = await runPush([...DEFAULT_ARGS, '-y'], { + TEST_OVERRIDE: false, + }); + expect(output).toContain('Push command Aborted'); + }); + + it('delete entire project with --yes flag', async () => { + const output = await runPush([...DEFAULT_ARGS, '-y']); + expect(output).toContain(deleteProgress); + }); + + it('delete entire project with prompt override', async () => { + const output = await runPush([...DEFAULT_ARGS, '-y'], { + TEST_OVERRIDE: true, + }); + expect(output).toContain(deleteProgress); + }); + + it('push journeys', async () => { + const testJourney = join(PROJECT_DIR, 'test.journey.ts'); + await writeFile( + testJourney, + `import {journey, monitor} from '../../../src/index'; + journey('journey 1', () => monitor.use({ id: 'j1' })); + journey('journey 2', () => monitor.use({ id: 'j2' }));` + ); + const output = await runPush(); + expect(output).toContain('Pushing monitors for project: test-project'); + expect(output).toContain('bundling 2 monitors'); + expect(output).toContain('creating or updating 2 monitors'); + expect(output).toContain(deleteProgress); + expect(output).toContain('✓ Pushed:'); + await rm(testJourney, { force: true }); + }); }); }); }); diff --git a/__tests__/push/monitor.test.ts b/__tests__/push/monitor.test.ts index 5f231430..5e57bf42 100644 --- a/__tests__/push/monitor.test.ts +++ b/__tests__/push/monitor.test.ts @@ -28,15 +28,14 @@ import { join } from 'path'; import { generateTempPath } from '../../src/helpers'; import { buildMonitorSchema, - createMonitors, createLightweightMonitors, + diffMonitors, parseSchedule, } from '../../src/push/monitor'; import { Server } from '../utils/server'; import { createTestMonitor } from '../utils/test-config'; describe('Monitors', () => { - const monitor = createTestMonitor('example.journey.ts'); let server: Server; beforeAll(async () => { server = await Server.create(); @@ -48,29 +47,55 @@ describe('Monitors', () => { process.env.NO_COLOR = ''; }); + it('diff monitors', () => { + const local = [ + { journey_id: 'j1', hash: 'hash1' }, + { journey_id: 'j2', hash: 'hash2' }, + { journey_id: 'j3', hash: 'hash3' }, + { journey_id: 'j4', hash: 'hash4' }, + ]; + const remote = [ + { journey_id: 'j1', hash: 'hash1' }, + { journey_id: 'j2', hash: 'hash2-changed' }, + { journey_id: 'j4', hash: '' }, // Hash reset in UI + { journey_id: 'j5', hash: 'hash5' }, + ]; + const result = diffMonitors(local, remote); + expect(Array.from(result.newIDs)).toEqual(['j3']); + expect(Array.from(result.changedIDs)).toEqual(['j2', 'j4']); + expect(Array.from(result.removedIDs)).toEqual(['j5']); + expect(Array.from(result.unchangedIDs)).toEqual(['j1']); + }); + it('build lightweight monitor schema', async () => { - const schema = await buildMonitorSchema([ - createTestMonitor('heartbeat.yml', 'http'), - ]); + const schema = await buildMonitorSchema( + [createTestMonitor('heartbeat.yml', 'http')], + true + ); expect(schema[0]).toEqual({ id: 'test-monitor', name: 'test', schedule: 10, type: 'http', enabled: true, + hash: 'fS9hbiqcopwk3gMeDrqMgyp0mEpeyFWy6F/YFSnfWPE=', locations: ['europe-west2-a', 'australia-southeast1-a'], privateLocations: ['germany'], }); }); it('build browser monitor schema', async () => { - const schema = await buildMonitorSchema([monitor]); + const schema = await buildMonitorSchema( + [createTestMonitor('example.journey.ts')], + true + ); expect(schema[0]).toEqual({ id: 'test-monitor', name: 'test', schedule: 10, type: 'browser', enabled: true, + hash: 'I+bYcE74C35IaKpHeN04wBfinO3qEiqlcaksKFAmkBg=', locations: ['europe-west2-a', 'australia-southeast1-a'], privateLocations: ['germany'], content: expect.any(String), @@ -97,40 +122,6 @@ describe('Monitors', () => { expect(parseSchedule('@every 10h2m10s')).toBe(60); }); - it('api schema', async () => { - server.route( - '/s/dummy/api/synthetics/service/project/monitors', - (req, res) => { - let data = ''; - req.on('data', chunks => { - data += chunks; - }); - req.on('end', () => { - // Write the post data back - res.end(data.toString()); - }); - } - ); - const schema = await buildMonitorSchema([monitor]); - const { statusCode, body } = await createMonitors( - schema, - { - url: `${server.PREFIX}`, - auth: 'apiKey', - id: 'blah', - space: 'dummy', - }, - false - ); - - expect(statusCode).toBe(200); - expect(await body.json()).toEqual({ - project: 'blah', - keep_stale: false, - monitors: schema, - }); - }); - describe('Lightweight monitors', () => { const PROJECT_DIR = generateTempPath(); const HB_SOURCE = join(PROJECT_DIR, 'heartbeat.yml'); diff --git a/__tests__/push/request.test.ts b/__tests__/push/request.test.ts index 53305c9b..27c844f4 100644 --- a/__tests__/push/request.test.ts +++ b/__tests__/push/request.test.ts @@ -47,7 +47,7 @@ describe('Push api request', () => { it('format 404 error', () => { const message = 'Not Found'; - expect(formatNotFoundError(message)).toMatchSnapshot(); + expect(formatNotFoundError('http://foo', message)).toMatchSnapshot(); }); it('format failed monitors', () => { diff --git a/__tests__/utils/kibana-test-server.ts b/__tests__/utils/kibana-test-server.ts new file mode 100644 index 00000000..cb323402 --- /dev/null +++ b/__tests__/utils/kibana-test-server.ts @@ -0,0 +1,87 @@ +/** + * MIT License + * + * Copyright (c) 2020-present, Elastic NV + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ +import { Server } from './server'; +import { + GetResponse, + LegacyAPISchema, + PutResponse, +} from '../../src/push/kibana_api'; + +export const createKibanaTestServer = async (kibanaVersion: string) => { + const server = await Server.create({ port: 54455 }); + server.route('/s/dummy/api/status', (req, res) => + res.end(JSON.stringify({ version: { number: kibanaVersion } })) + ); + // Legacy + server.route( + '/s/dummy/api/synthetics/service/project/monitors', + async (req, res) => { + await new Promise(r => setTimeout(r, 20)); + req.on('data', chunks => { + const schema = JSON.parse(chunks.toString()) as LegacyAPISchema; + res.write( + JSON.stringify( + schema.monitors.length + ' monitors created successfully' + ) + '\n' + ); + if (!schema.keep_stale) { + // write more than the stream buffer to check the broken NDJSON data + res.write( + JSON.stringify(Buffer.from('a'.repeat(70000)).toString()) + '\n' + ); + } + }); + req.on('end', () => { + const apiRes = { failedMonitors: [], failedStaleMonitors: [] }; + res.end(JSON.stringify(apiRes)); + }); + } + ); + + // Post 8.6 + const basePath = '/s/dummy/api/synthetics/project/test-project/monitors'; + server.route(basePath, (req, res) => { + const getResp = { + total: 2, + monitors: [ + { journey_id: 'j3', hash: 'hash1' }, + { journey_id: 'j4', hash: 'hash2' }, + ], + } as GetResponse; + res.end(JSON.stringify(getResp)); + }); + server.route(basePath + '/_bulk_update', (req, res) => { + const updateResponse = { + createdMonitors: ['j1', 'j2'], + updatedMonitors: [], + failedMonitors: [], + } as PutResponse; + res.end(JSON.stringify(updateResponse)); + }); + server.route(basePath + '/_bulk_delete', (req, res) => { + res.end(JSON.stringify({ deleted_monitors: ['j3', 'j4'] })); + }); + return server; +}; diff --git a/package-lock.json b/package-lock.json index 75060b6b..3abe5a1c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,6 +22,7 @@ "micromatch": "^4.0.5", "playwright-chromium": "=1.27.1", "playwright-core": "=1.27.1", + "semver": "^7.3.8", "sharp": "^0.31.1", "snakecase-keys": "^4.0.1", "sonic-boom": "^3.2.0", @@ -5886,9 +5887,9 @@ "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" }, "node_modules/semver": { - "version": "7.3.7", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.7.tgz", - "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", + "version": "7.3.8", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", + "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", "dependencies": { "lru-cache": "^6.0.0" }, @@ -11043,9 +11044,9 @@ "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" }, "semver": { - "version": "7.3.7", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.7.tgz", - "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", + "version": "7.3.8", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", + "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", "requires": { "lru-cache": "^6.0.0" } diff --git a/package.json b/package.json index 6c715ebf..614a8f1e 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "micromatch": "^4.0.5", "playwright-chromium": "=1.27.1", "playwright-core": "=1.27.1", + "semver": "^7.3.8", "sharp": "^0.31.1", "snakecase-keys": "^4.0.1", "sonic-boom": "^3.2.0", diff --git a/src/core/runner.ts b/src/core/runner.ts index 770ced07..082c6459 100644 --- a/src/core/runner.ts +++ b/src/core/runner.ts @@ -403,9 +403,7 @@ export default class Runner { if (!journey.isMatch(match, tags)) { continue; } - this.#currentJourney = journey; - /** * Execute dummy callback to get all monitor specific * configurations for the current journey diff --git a/src/dsl/monitor.ts b/src/dsl/monitor.ts index 510b017a..bfe19dc9 100644 --- a/src/dsl/monitor.ts +++ b/src/dsl/monitor.ts @@ -23,6 +23,7 @@ * */ +import { createHash } from 'crypto'; import merge from 'deepmerge'; import { bold, red } from 'kleur/colors'; import { @@ -94,6 +95,11 @@ export class Monitor { this.filter = filter; } + hash(): string { + const hash = createHash('sha256'); + return hash.update(JSON.stringify(this.config)).digest('base64'); + } + validate() { const schedule = this.config.schedule; if (ALLOWED_SCHEDULES.includes(schedule)) { diff --git a/src/helpers.ts b/src/helpers.ts index c4d4dfc6..e33193d2 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -339,16 +339,27 @@ export function safeNDJSONParse(data: string | string[]) { } // Console helpers -export function write(message: string) { - process.stderr.write(message + '\n'); +export function write(message: string, live?: boolean) { + process.stderr.write(message + (live ? '\r' : '\n')); } export function progress(message: string) { write(cyan(bold(`${symbols.progress} ${message}`))); } -export function apiProgress(message: string) { - write(grey(`> ${message}`)); +export async function liveProgress(promise: Promise, message: string) { + const start = now(); + const interval = setInterval(() => { + apiProgress(`${message} (${Math.trunc(now() - start)}ms)`, true); + }, 500); + promise.finally(() => clearInterval(interval)); + const result = await promise; + apiProgress(`${message} (${Math.trunc(now() - start)}ms)`); + return result; +} + +export function apiProgress(message: string, live = false) { + write(grey(`> ${message}`), live); } export function error(message: string) { @@ -370,11 +381,3 @@ export function removeTrailingSlash(url: string) { export function getMonitorManagementURL(url) { return removeTrailingSlash(url) + '/app/uptime/manage-monitors/all'; } - -export function getArrayChunks(arr: T[], chunkSize: number): T[][] { - const chunks = []; - for (let i = 0; i < arr.length; i += chunkSize) { - chunks.push(arr.slice(i, i + chunkSize)); - } - return chunks; -} diff --git a/src/locations/index.ts b/src/locations/index.ts index a0296b71..1972e258 100644 --- a/src/locations/index.ts +++ b/src/locations/index.ts @@ -23,13 +23,9 @@ * */ -import { - formatAPIError, - formatNotFoundError, - sendRequest, - ok, -} from '../push/request'; -import { indent, removeTrailingSlash, write } from '../helpers'; +import { sendReqAndHandleError } from '../push/request'; +import { indent, write } from '../helpers'; +import { generateURL } from '../push/utils'; export type LocationCmdOptions = { auth: string; @@ -49,21 +45,12 @@ export type LocationAPIResponse = { const PRIVATE_KEYWORD = '(private)'; export async function getLocations(options: LocationCmdOptions) { - const url = - removeTrailingSlash(options.url) + '/internal/uptime/service/locations'; - const { body, statusCode } = await sendRequest({ - url, + const resp = await sendReqAndHandleError({ + url: generateURL(options, 'location'), method: 'GET', auth: options.auth, }); - if (statusCode === 404) { - throw formatNotFoundError(await body.text()); - } - if (!ok(statusCode)) { - const { error, message } = await body.json(); - throw formatAPIError(statusCode, error, message); - } - return ((await body.json()) as LocationAPIResponse).locations; + return resp.locations; } export function formatLocations(locations: Array) { diff --git a/src/push/index.ts b/src/push/index.ts index 5e143c7b..93736eea 100644 --- a/src/push/index.ts +++ b/src/push/index.ts @@ -22,118 +22,107 @@ * THE SOFTWARE. * */ - +import semver from 'semver'; import { readFile, writeFile } from 'fs/promises'; import { prompt } from 'enquirer'; import { bold, grey } from 'kleur/colors'; import { - ok, - formatAPIError, - formatFailedMonitors, - formatNotFoundError, - formatStaleMonitors, -} from './request'; -import { buildMonitorSchema, createMonitors, MonitorSchema } from './monitor'; + getLocalMonitors, + buildMonitorSchema, + diffMonitors as diffMonitorHashIDs, + MonitorSchema, +} from './monitor'; import { ALLOWED_SCHEDULES, Monitor } from '../dsl/monitor'; import { progress, - apiProgress, + liveProgress, write, error, warn, indent, - safeNDJSONParse, done, getMonitorManagementURL, - getArrayChunks, } from '../helpers'; import type { PushOptions, ProjectSettings } from '../common_types'; import { findSyntheticsConfig, readConfig } from '../config'; +import { + bulkDeleteMonitors, + bulkGetMonitors, + bulkPutMonitors, + getVersion, + createMonitorsLegacy, + CHUNK_SIZE, +} from './kibana_api'; +import { getChunks, logDiff } from './utils'; export async function push(monitors: Monitor[], options: PushOptions) { - let schemas: MonitorSchema[] = []; - if (monitors.length > 0) { - const duplicates = trackDuplicates(monitors); - if (duplicates.size > 0) { - throw error(formatDuplicateError(duplicates)); - } + const duplicates = trackDuplicates(monitors); + if (duplicates.size > 0) { + throw error(formatDuplicateError(duplicates)); + } + progress(`Pushing monitors for project: ${options.id}`); - progress(`preparing all monitors`); - schemas = await buildMonitorSchema(monitors); + const stackVersion = await getVersion(options); + const isV2 = semver.satisfies(stackVersion, '>=8.6.0'); + if (!isV2) { + return await pushLegacy(monitors, options, stackVersion); + } + + const local = getLocalMonitors(monitors); + const { monitors: remote } = await bulkGetMonitors(options); + const { newIDs, changedIDs, removedIDs, unchangedIDs } = diffMonitorHashIDs( + local, + remote + ); + logDiff(newIDs, changedIDs, removedIDs, unchangedIDs); - progress(`creating all monitors`); - const chunks = getArrayChunks(schemas, 10); + const updatedMonitors = new Set([...changedIDs, ...newIDs]); + if (updatedMonitors.size > 0) { + const toBundle = monitors.filter(m => updatedMonitors.has(m.config.id)); + progress(`bundling ${toBundle.length} monitors`); + const schemas = await buildMonitorSchema(toBundle, true); + const chunks = getChunks(schemas, CHUNK_SIZE); for (const chunk of chunks) { - await pushMonitors({ schemas: chunk, keepStale: true, options }); + await liveProgress( + bulkPutMonitors(options, chunk), + `creating or updating ${chunk.length} monitors` + ); } - } else { - write(''); - const { deleteAll } = await prompt<{ deleteAll: boolean }>({ - type: 'confirm', - skip() { - if (options.yes) { - this.initial = process.env.TEST_OVERRIDE ?? true; - return true; - } - return false; - }, - name: 'deleteAll', - message: `Pushing without any monitors will delete all monitors associated with the project.\n Do you want to continue?`, - initial: false, - }); - if (!deleteAll) { - throw warn('Push command Aborted'); + } + + if (removedIDs.size > 0) { + if (updatedMonitors.size === 0 && unchangedIDs.size === 0) { + await promptConfirmDeleteAll(options); + } + const chunks = getChunks(Array.from(removedIDs), CHUNK_SIZE); + for (const chunk of chunks) { + await liveProgress( + bulkDeleteMonitors(options, chunk), + `deleting ${chunk.length} monitors` + ); } } - progress(`deleting all stale monitors`); - await pushMonitors({ schemas, keepStale: false, options }); done(`Pushed: ${grey(getMonitorManagementURL(options.url))}`); } -export async function pushMonitors({ - schemas, - keepStale, - options, -}: { - schemas: MonitorSchema[]; - keepStale: boolean; - options: PushOptions; -}) { - const { body, statusCode } = await createMonitors( - schemas, - options, - keepStale - ); - if (statusCode === 404) { - throw formatNotFoundError(await body.text()); - } - if (!ok(statusCode)) { - const { error, message } = await body.json(); - throw formatAPIError(statusCode, error, message); - } - const allchunks = []; - for await (const data of body) { - allchunks.push(Buffer.from(data)); - } - const chunks = safeNDJSONParse(Buffer.concat(allchunks).toString('utf-8')); - // Its kind of hacky for now where Kibana streams the response by - // writing the data as NDJSON events (data can be interleaved), we - // distinguish the final data by checking if the event was a progress vs complete event - for (const chunk of chunks) { - if (typeof chunk === 'string') { - // TODO: add progress back for all states once we get the fix - // on kibana side - keepStale && apiProgress(chunk); - continue; - } - const { failedMonitors, failedStaleMonitors } = chunk; - if (failedMonitors && failedMonitors.length > 0) { - throw formatFailedMonitors(failedMonitors); - } - if (failedStaleMonitors.length > 0) { - throw formatStaleMonitors(failedStaleMonitors); - } +async function promptConfirmDeleteAll(options: PushOptions) { + write(''); + const { deleteAll } = await prompt<{ deleteAll: boolean }>({ + type: 'confirm', + skip() { + if (options.yes) { + this.initial = process.env.TEST_OVERRIDE ?? true; + return true; + } + return false; + }, + name: 'deleteAll', + message: `Pushing without any monitors will delete all monitors associated with the project.\n Do you want to continue?`, + initial: false, + }); + if (!deleteAll) { + throw warn('Push command Aborted'); } } @@ -174,7 +163,7 @@ export async function loadSettings() { } return config.project || ({} as ProjectSettings); } catch (e) { - throw error(`Aborted (missing synthetics config file), Project not set up corrrectly. + throw error(`Aborted (missing synthetics config file), Project not set up correctly. ${INSTALLATION_HELP}`); } @@ -254,3 +243,40 @@ export async function catchIncorrectSettings( await overrideSettings(settings.id, options.id); } } + +export async function pushLegacy( + monitors: Monitor[], + options: PushOptions, + version: number +) { + const noLightWeightSupport = semver.satisfies(version, '<8.5.0'); + if ( + noLightWeightSupport && + monitors.some(monitor => monitor.type !== 'browser') + ) { + throw error( + `Aborted: Lightweight monitors are not supported in ${version}. Please upgrade to 8.5.0 or above.` + ); + } + + let schemas: MonitorSchema[] = []; + if (monitors.length > 0) { + progress(`bundling ${monitors.length} monitors`); + schemas = await buildMonitorSchema(monitors, false); + const chunks = getChunks(schemas, 10); + for (const chunk of chunks) { + await liveProgress( + createMonitorsLegacy({ schemas: chunk, keepStale: true, options }), + `creating or updating ${chunk.length} monitors` + ); + } + } else { + await promptConfirmDeleteAll(options); + } + await liveProgress( + createMonitorsLegacy({ schemas, keepStale: false, options }), + `deleting all stale monitors` + ); + + done(`Pushed: ${grey(getMonitorManagementURL(options.url))}`); +} diff --git a/src/push/kibana_api.ts b/src/push/kibana_api.ts new file mode 100644 index 00000000..8102a565 --- /dev/null +++ b/src/push/kibana_api.ts @@ -0,0 +1,180 @@ +/** + * MIT License + * + * Copyright (c) 2020-present, Elastic NV + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +import { PushOptions } from '../common_types'; +import { safeNDJSONParse } from '../helpers'; +import { MonitorHashID, MonitorSchema } from './monitor'; +import { + formatFailedMonitors, + formatStaleMonitors, + handleError, + sendReqAndHandleError, + sendRequest, + APIMonitorError, +} from './request'; +import { generateURL } from './utils'; + +// Default chunk size for bulk put / delete +export const CHUNK_SIZE = 100; + +export type PutResponse = { + createdMonitors: string[]; + updatedMonitors: string[]; + failedMonitors: APIMonitorError[]; +}; + +export async function bulkPutMonitors( + options: PushOptions, + schemas: MonitorSchema[] +) { + const resp = await sendReqAndHandleError({ + url: generateURL(options, 'bulk_update') + '/_bulk_update', + method: 'PUT', + auth: options.auth, + body: JSON.stringify({ monitors: schemas }), + }); + const { failedMonitors } = resp; + if (failedMonitors && failedMonitors.length > 0) { + throw formatFailedMonitors(failedMonitors); + } + return resp; +} + +export type GetResponse = { + total: number; + monitors: MonitorHashID[]; + after_key?: string; +}; + +export async function bulkGetMonitors( + options: PushOptions +): Promise { + let afterKey = null; + let total = 0; + const monitors: MonitorHashID[] = []; + let url = generateURL(options, 'bulk_get'); + do { + if (afterKey) { + url += `?search_after=${afterKey}`; + } + const resp = await sendReqAndHandleError({ + url, + method: 'GET', + auth: options.auth, + }); + afterKey = resp.after_key; + + // The first page gives the total number of monitors + if (total == 0) { + total = resp.total; + } + monitors.push(...resp.monitors); + } while (afterKey); + + return { total, monitors }; +} + +export type DeleteResponse = { + deleted_monitors: string[]; +}; + +export async function bulkDeleteMonitors( + options: PushOptions, + monitorIDs: string[] +) { + return await sendReqAndHandleError({ + url: generateURL(options, 'bulk_delete') + '/_bulk_delete', + method: 'DELETE', + auth: options.auth, + body: JSON.stringify({ monitors: monitorIDs }), + }); +} + +type StatusResponse = { + version: { + number: number; + }; +}; + +export async function getVersion(options: PushOptions) { + const data = await sendReqAndHandleError({ + url: generateURL(options, 'status'), + method: 'GET', + auth: options.auth, + }); + return data.version.number; +} + +export type LegacyAPISchema = { + project: string; + keep_stale: boolean; + monitors: MonitorSchema[]; +}; + +export async function createMonitorsLegacy({ + schemas, + keepStale, + options, +}: { + schemas: MonitorSchema[]; + keepStale: boolean; + options: PushOptions; +}) { + const schema: LegacyAPISchema = { + project: options.id, + keep_stale: keepStale, + monitors: schemas, + }; + const url = generateURL(options, 'legacy'); + const { body, statusCode } = await sendRequest({ + url, + method: 'PUT', + auth: options.auth, + body: JSON.stringify(schema), + }); + + const resBody = await handleError(statusCode, url, body); + const allchunks = []; + for await (const data of resBody) { + allchunks.push(Buffer.from(data)); + } + const chunks = safeNDJSONParse(Buffer.concat(allchunks).toString('utf-8')); + // Its kind of hacky for now where Kibana streams the response by + // writing the data as NDJSON events (data can be interleaved), we + // distinguish the final data by checking if the event was a progress vs complete event + for (const chunk of chunks) { + if (typeof chunk === 'string') { + // Ignore the progress from Kibana as we chunk the requests + continue; + } + const { failedMonitors, failedStaleMonitors } = chunk; + if (failedMonitors && failedMonitors.length > 0) { + throw formatFailedMonitors(failedMonitors); + } + if (failedStaleMonitors.length > 0) { + throw formatStaleMonitors(failedStaleMonitors); + } + } +} diff --git a/src/push/monitor.ts b/src/push/monitor.ts index 64155a04..750bd6f7 100644 --- a/src/push/monitor.ts +++ b/src/push/monitor.ts @@ -28,14 +28,7 @@ import { join } from 'path'; import { LineCounter, parseDocument, YAMLSeq, YAMLMap } from 'yaml'; import { bold, red } from 'kleur/colors'; import { Bundler } from './bundler'; -import { sendRequest } from './request'; -import { - removeTrailingSlash, - SYNTHETICS_PATH, - totalist, - indent, - warn, -} from '../helpers'; +import { SYNTHETICS_PATH, totalist, indent, warn } from '../helpers'; import { LocationsMap } from '../locations/public-locations'; import { ALLOWED_SCHEDULES, Monitor, MonitorConfig } from '../dsl/monitor'; import { PushOptions } from '../common_types'; @@ -44,12 +37,14 @@ export type MonitorSchema = Omit & { locations: string[]; content?: string; filter?: Monitor['filter']; + hash?: string; }; -export type APISchema = { - project: string; - keep_stale: boolean; - monitors: MonitorSchema[]; +// Abbreviated monitor info, as often returned by the API, +// just the journey ID and hash +export type MonitorHashID = { + journey_id?: string; + hash?: string; }; function translateLocation(locations?: MonitorConfig['locations']) { @@ -57,7 +52,67 @@ function translateLocation(locations?: MonitorConfig['locations']) { return locations.map(loc => LocationsMap[loc] || loc).filter(Boolean); } -export async function buildMonitorSchema(monitors: Monitor[]) { +class RemoteDiffResult { + // The set of monitor IDs that have been added + newIDs = new Set(); + // Monitor IDs that are different locally than remotely + changedIDs = new Set(); + // Monitor IDs that are no longer present locally + removedIDs = new Set(); + // Monitor IDs that are identical on the remote server + unchangedIDs = new Set(); +} + +export function diffMonitors( + local: MonitorHashID[], + remote: MonitorHashID[] +): RemoteDiffResult { + const result = new RemoteDiffResult(); + const localMonitorsIDToHash = new Map(); + for (const hashID of local) { + localMonitorsIDToHash.set(hashID.journey_id, hashID.hash); + } + const remoteMonitorsIDToHash = new Map(); + for (const hashID of remote) { + remoteMonitorsIDToHash.set(hashID.journey_id, hashID.hash); + } + + // Compare local to remote + for (const [localID, localHash] of localMonitorsIDToHash) { + // Hash is reset to '' when a monitor is edited on the UI + if (!remoteMonitorsIDToHash.has(localID)) { + result.newIDs.add(localID); + } else { + const remoteHash = remoteMonitorsIDToHash.get(localID); + if (remoteHash != localHash) { + result.changedIDs.add(localID); + } else if (remoteHash === localHash) { + result.unchangedIDs.add(localID); + } + } + // We no longer need to process this ID, removing it here + // reduces the numbers considered in the next phase + remoteMonitorsIDToHash.delete(localID); + } + + for (const [id] of remoteMonitorsIDToHash) { + result.removedIDs.add(id); + } + return result; +} + +export function getLocalMonitors(monitors: Monitor[]) { + const data: MonitorHashID[] = []; + for (const monitor of monitors) { + data.push({ + journey_id: monitor.config.id, + hash: monitor.hash(), + }); + } + return data; +} + +export async function buildMonitorSchema(monitors: Monitor[], isV2: boolean) { /** * Set up the bundle artifacts path which can be used to * create the bundles required for uploading journeys @@ -69,10 +124,13 @@ export async function buildMonitorSchema(monitors: Monitor[]) { for (const monitor of monitors) { const { source, config, filter, type } = monitor; - const schema = { + const schema: MonitorSchema = { ...config, locations: translateLocation(config.locations), }; + if (isV2) { + schema.hash = monitor.hash(); + } if (type === 'browser') { const outPath = join(bundlePath, config.name + '.zip'); const content = await bundler.build(source.file, outPath); @@ -216,24 +274,3 @@ export function nearestSchedule(schedule: number) { } return ALLOWED_SCHEDULES[end]; } - -export async function createMonitors( - monitors: MonitorSchema[], - options: PushOptions, - keepStale: boolean -) { - const schema: APISchema = { - project: options.id, - keep_stale: keepStale, - monitors, - }; - - return await sendRequest({ - url: - removeTrailingSlash(options.url) + - `/s/${options.space}/api/synthetics/service/project/monitors`, - method: 'PUT', - auth: options.auth, - body: JSON.stringify(schema), - }); -} diff --git a/src/push/request.ts b/src/push/request.ts index 864b6175..db77c0df 100644 --- a/src/push/request.ts +++ b/src/push/request.ts @@ -24,7 +24,8 @@ */ import { bold, red, yellow } from 'kleur/colors'; -import { Dispatcher, request } from 'undici'; +import type { Dispatcher } from 'undici'; +import { request } from 'undici'; import { indent, symbols } from '../helpers'; /* eslint-disable @typescript-eslint/no-var-requires */ @@ -51,6 +52,40 @@ export async function sendRequest(options: APIRequestOptions) { }); } +export async function sendReqAndHandleError( + options: APIRequestOptions +): Promise { + const { statusCode, body } = await sendRequest(options); + return await (await handleError(statusCode, options.url, body)).json(); +} + +// Handle bad status code errors from Kibana API and format the +// error message to be displayed to the user. +// returns the response stream if no error is found +export async function handleError( + statusCode: number, + url: string, + body: Dispatcher.ResponseData['body'] +): Promise { + if (statusCode === 404) { + throw formatNotFoundError(url, await body.text()); + } else if (!ok(statusCode)) { + let parsed: { error: string; message: string }; + try { + parsed = await body.json(); + } catch (e) { + throw formatAPIError( + statusCode, + 'unexpected non-JSON error', + await body.text() + ); + } + throw formatAPIError(statusCode, parsed.error, parsed.message); + } + + return body; +} + export function ok(statusCode: number) { return statusCode >= 200 && statusCode <= 299; } @@ -61,10 +96,10 @@ export type APIMonitorError = { details: string; }; -export function formatNotFoundError(message: string) { +export function formatNotFoundError(url: string, message: string) { return red( bold( - `${symbols['failed']} Please check your kibana url and try again - 404:${message}` + `${symbols['failed']} Please check your kibana url: ${url} and try again - 404:${message}` ) ); } diff --git a/src/push/utils.ts b/src/push/utils.ts new file mode 100644 index 00000000..a9d4f591 --- /dev/null +++ b/src/push/utils.ts @@ -0,0 +1,77 @@ +/** + * MIT License + * + * Copyright (c) 2020-present, Elastic NV + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +import { progress, removeTrailingSlash } from '../helpers'; +import { green, red, grey, yellow } from 'kleur/colors'; +import { PushOptions } from '../common_types'; + +export function logDiff>( + newIDs: T, + changedIDs: T, + removedIDs: T, + unchangedIDs: T +) { + progress( + 'Monitor Diff: ' + + green(`Added(${newIDs.size}) `) + + yellow(`Updated(${changedIDs.size}) `) + + red(`Removed(${removedIDs.size}) `) + + grey(`Unchanged(${unchangedIDs.size})`) + ); +} + +export function getChunks(arr: any[], size: number) { + const chunks = []; + for (let i = 0; i < arr.length; i += size) { + chunks.push(arr.slice(i, i + size)); + } + return chunks; +} + +type Operation = + | 'status' + | 'bulk_get' + | 'bulk_update' + | 'bulk_delete' + | 'legacy' + | 'location'; + +export function generateURL(options: PushOptions, operation: Operation) { + const url = removeTrailingSlash(options.url); + switch (operation) { + case 'status': + return `${url}/s/${options.space}/api/status`; + case 'bulk_get': + case 'bulk_update': + case 'bulk_delete': + return `${url}/s/${options.space}/api/synthetics/project/${options.id}/monitors`; + case 'legacy': + return `${url}/s/${options.space}/api/synthetics/service/project/monitors`; + case 'location': + return `${url}/internal/uptime/service/locations`; + default: + throw new Error('Invalid operation'); + } +}