diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96f72dff..9e345ac8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: 18.x cache: 'yarn' - run: yarn --frozen-lockfile - run: yarn build @@ -25,7 +25,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: 18.x cache: 'yarn' - run: yarn --frozen-lockfile - run: yarn build @@ -36,7 +36,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: 18.x cache: 'yarn' - run: yarn --frozen-lockfile - run: yarn build @@ -48,7 +48,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: 18.x cache: 'yarn' - run: yarn --frozen-lockfile - run: yarn build diff --git a/.github/workflows/release-canary.yml b/.github/workflows/release-canary.yml index 33a918fb..069829dc 100644 --- a/.github/workflows/release-canary.yml +++ b/.github/workflows/release-canary.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: 18.x cache: 'yarn' - run: yarn --frozen-lockfile diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d1c17534..34835104 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: 18.x cache: 'yarn' - run: yarn --frozen-lockfile diff --git a/packages/backend-tools/CHANGELOG.md b/packages/backend-tools/CHANGELOG.md index f9476a18..896204d9 100644 --- a/packages/backend-tools/CHANGELOG.md +++ b/packages/backend-tools/CHANGELOG.md @@ -1,5 +1,11 @@ # @l2beat/backend-tools +## 0.6.0 + +### Minor Changes + +- Refactor logger to allow for multiple backends and formatters + ## 0.5.2 ### Patch Changes diff --git a/packages/backend-tools/package.json b/packages/backend-tools/package.json index 9d9f588e..6cbb137e 100644 --- a/packages/backend-tools/package.json +++ b/packages/backend-tools/package.json @@ -1,7 +1,7 @@ { "name": "@l2beat/backend-tools", "description": "Common utilities for L2BEAT projects.", - "version": "0.5.2", + "version": "0.6.0", "license": "MIT", "repository": "https://github.com/l2beat/tools", "bugs": { @@ -30,12 +30,16 @@ "typecheck": "tsc --noEmit" }, "dependencies": { + "@elastic/elasticsearch": "^8.13.1", "chalk": "^4.1.2", "dotenv": "^16.3.1", - "error-stack-parser": "^2.1.4" + "error-stack-parser": "^2.1.4", + "uuid": "^9.0.1" }, "devDependencies": { "@sinonjs/fake-timers": "^11.1.0", - "@types/sinonjs__fake-timers": "^8.1.2" + "@types/elasticsearch": "^5.0.43", + "@types/sinonjs__fake-timers": "^8.1.2", + "@types/uuid": "^9.0.8" } } diff --git a/packages/backend-tools/src/elastic-search/ElasticSearchBackend.test.ts b/packages/backend-tools/src/elastic-search/ElasticSearchBackend.test.ts new file mode 100644 index 00000000..8423c32a --- /dev/null +++ b/packages/backend-tools/src/elastic-search/ElasticSearchBackend.test.ts @@ -0,0 +1,91 @@ +import { expect, mockFn, MockObject, mockObject } from 'earl' + +import { + ElasticSearchBackend, + ElasticSearchBackendOptions, + UuidProvider, +} from './ElasticSearchBackend' +import { ElasticSearchClient } from './ElasticSearchClient' + +const flushInterval = 10 +const id = 'some-id' +const indexPrefix = 'logs-' +const indexName = createIndexName() +const log = { + '@timestamp': '2024-04-24T21:02:30.916Z', + log: { + level: 'INFO', + }, + message: 'Update started', +} + +describe(ElasticSearchBackend.name, () => { + it("creates index if doesn't exist", async () => { + const clientMock = createClienMock(false) + const backendMock = createBackendMock(clientMock) + + backendMock.log(JSON.stringify(log)) + + // wait for log flus + await delay(flushInterval + 10) + + expect(clientMock.indexExist).toHaveBeenOnlyCalledWith(indexName) + expect(clientMock.indexCreate).toHaveBeenOnlyCalledWith(indexName) + }) + + it('does nothing if buffer is empty', async () => { + const clientMock = createClienMock(false) + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const backendMock = createBackendMock(clientMock) + + // wait for log flush + await delay(flushInterval + 10) + + expect(clientMock.bulk).not.toHaveBeenCalled() + }) + + it('pushes logs to ES if there is something in the buffer', async () => { + const clientMock = createClienMock(false) + const backendMock = createBackendMock(clientMock) + + backendMock.log(JSON.stringify(log)) + + // wait for log flush + await delay(flushInterval + 10) + + expect(clientMock.bulk).toHaveBeenOnlyCalledWith( + [{ id, ...log }], + indexName, + ) + }) +}) + +function createClienMock(indextExist = true) { + return mockObject({ + indexExist: mockFn(async (_: string): Promise => indextExist), + indexCreate: mockFn(async (_: string): Promise => {}), + bulk: mockFn(async (_: object[]): Promise => true), + }) +} + +function createBackendMock(clientMock: MockObject) { + const uuidProviderMock: UuidProvider = () => id + + const options: ElasticSearchBackendOptions = { + node: 'node', + apiKey: 'apiKey', + indexPrefix, + flushInterval, + } + + return new ElasticSearchBackend(options, clientMock, uuidProviderMock) +} + +function createIndexName() { + const now = new Date() + return `${indexPrefix}-${now.getFullYear()}.${now.getMonth()}.${now.getDay()}` +} + +async function delay(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)) +} diff --git a/packages/backend-tools/src/elastic-search/ElasticSearchBackend.ts b/packages/backend-tools/src/elastic-search/ElasticSearchBackend.ts new file mode 100644 index 00000000..f1b0245a --- /dev/null +++ b/packages/backend-tools/src/elastic-search/ElasticSearchBackend.ts @@ -0,0 +1,102 @@ +import { v4 as uuidv4 } from 'uuid' + +import { LoggerBackend } from '../logger/interfaces' +import { + ElasticSearchClient, + ElasticSearchClientOptions, +} from './ElasticSearchClient' + +export interface ElasticSearchBackendOptions + extends ElasticSearchClientOptions { + flushInterval?: number + indexPrefix?: string +} + +export type UuidProvider = () => string + +export class ElasticSearchBackend implements LoggerBackend { + private readonly buffer: string[] + + constructor( + private readonly options: ElasticSearchBackendOptions, + private readonly client: ElasticSearchClient = new ElasticSearchClient( + options, + ), + private readonly uuidProvider: UuidProvider = uuidv4, + ) { + this.buffer = [] + this.start() + } + + public debug(message: string): void { + this.buffer.push(message) + } + + public log(message: string): void { + this.buffer.push(message) + } + + public warn(message: string): void { + this.buffer.push(message) + } + + public error(message: string): void { + this.buffer.push(message) + } + + private start(): void { + // eslint-disable-next-line @typescript-eslint/no-misused-promises + const interval = setInterval(async () => { + await this.flushLogs() + }, this.options.flushInterval ?? 10000) + + // object will not require the Node.js event loop to remain active + // nodejs.org/api/timers.html#timers_timeout_unref + interval.unref() + } + + private async flushLogs(): Promise { + if (!this.buffer.length) { + return + } + + try { + const index = await this.createIndex() + + // copy buffer contents as it may change during async operations below + const batch = [...this.buffer] + + //clear buffer + this.buffer.splice(0) + + const documents = batch.map( + (log) => + ({ + id: this.uuidProvider(), + ...JSON.parse(log), + } as object), + ) + + const success = await this.client.bulk(documents, index) + + if (!success) { + throw new Error('Failed to push liogs to Elastic Search node') + } + } catch (error) { + console.log(error) + } + } + + private async createIndex(): Promise { + const now = new Date() + const indexName = `${ + this.options.indexPrefix ?? 'logs-' + }-${now.getFullYear()}.${now.getMonth()}.${now.getDay()}` + + const exist = await this.client.indexExist(indexName) + if (!exist) { + await this.client.indexCreate(indexName) + } + return indexName + } +} diff --git a/packages/backend-tools/src/elastic-search/ElasticSearchClient.ts b/packages/backend-tools/src/elastic-search/ElasticSearchClient.ts new file mode 100644 index 00000000..b44252e4 --- /dev/null +++ b/packages/backend-tools/src/elastic-search/ElasticSearchClient.ts @@ -0,0 +1,40 @@ +import { Client } from '@elastic/elasticsearch' + +export interface ElasticSearchClientOptions { + node: string + apiKey: string +} + +// hides complexity of ElastiSearch client API +export class ElasticSearchClient { + private readonly client: Client + + constructor(private readonly options: ElasticSearchClientOptions) { + this.client = new Client({ + node: options.node, + auth: { + apiKey: options.apiKey, + }, + }) + } + + public async bulk(documents: object[], index: string): Promise { + const operations = documents.flatMap((doc: object) => [ + { index: { _index: index } }, + doc, + ]) + + const bulkResponse = await this.client.bulk({ refresh: true, operations }) + return bulkResponse.errors + } + + public async indexExist(index: string): Promise { + return await this.client.indices.exists({ index }) + } + + public async indexCreate(index: string): Promise { + await this.client.indices.create({ + index, + }) + } +} diff --git a/packages/backend-tools/src/index.ts b/packages/backend-tools/src/index.ts index e2bc4377..c1cc01d3 100644 --- a/packages/backend-tools/src/index.ts +++ b/packages/backend-tools/src/index.ts @@ -1,4 +1,9 @@ +export * from './elastic-search/ElasticSearchBackend' export * from './env' +export * from './logger/interfaces' +export * from './logger/LogFormatterEcs' +export * from './logger/LogFormatterJson' +export * from './logger/LogFormatterPretty' export * from './logger/Logger' export * from './rate-limit/RateLimiter' export * from './utils/assert' diff --git a/packages/backend-tools/src/logger/LogFormatterEcs.ts b/packages/backend-tools/src/logger/LogFormatterEcs.ts new file mode 100644 index 00000000..d11ef5d6 --- /dev/null +++ b/packages/backend-tools/src/logger/LogFormatterEcs.ts @@ -0,0 +1,31 @@ +import { LogEntry, LogFormatter } from './interfaces' +import { toJSON } from './toJSON' + +// https://www.elastic.co/guide/en/ecs/8.11/ecs-reference.html +export class LogFormatterEcs implements LogFormatter { + public format(entry: LogEntry): string { + const core = { + '@timestamp': entry.time.toISOString(), + log: { + level: entry.level, + }, + service: { + name: entry.service, + }, + message: entry.message, + error: entry.resolvedError + ? { + message: entry.resolvedError.error, + type: entry.resolvedError.name, + stack_trace: entry.resolvedError.stack, + } + : undefined, + } + + try { + return toJSON({ ...core, parameters: entry.parameters }) + } catch { + return toJSON({ ...core }) + } + } +} diff --git a/packages/backend-tools/src/logger/LogFormatterJson.ts b/packages/backend-tools/src/logger/LogFormatterJson.ts new file mode 100644 index 00000000..c7a20abd --- /dev/null +++ b/packages/backend-tools/src/logger/LogFormatterJson.ts @@ -0,0 +1,20 @@ +import { LogEntry, LogFormatter } from './interfaces' +import { toJSON } from './toJSON' + +export class LogFormatterJson implements LogFormatter { + public format(entry: LogEntry): string { + const core = { + time: entry.time.toISOString(), + level: entry.level, + service: entry.service, + message: entry.message, + error: entry.resolvedError, + } + + try { + return toJSON({ ...core, parameters: entry.parameters }) + } catch { + return toJSON({ ...core }) + } + } +} diff --git a/packages/backend-tools/src/logger/LogFormatterPretty.ts b/packages/backend-tools/src/logger/LogFormatterPretty.ts new file mode 100644 index 00000000..5300fbf1 --- /dev/null +++ b/packages/backend-tools/src/logger/LogFormatterPretty.ts @@ -0,0 +1,133 @@ +import chalk from 'chalk' +import { inspect } from 'util' + +import { LogEntry, LogFormatter } from './interfaces' +import { LogLevel } from './LogLevel' +import { toJSON } from './toJSON' + +const STYLES = { + bigint: 'white', + boolean: 'white', + date: 'white', + module: 'white', + name: 'blue', + null: 'white', + number: 'white', + regexp: 'white', + special: 'white', + string: 'white', + symbol: 'white', + undefined: 'white', +} + +const INDENT_SIZE = 4 +const INDENT = ' '.repeat(INDENT_SIZE) + +export class LogFormatterPretty implements LogFormatter { + constructor( + private readonly colors: boolean, + private readonly utc: boolean, + ) {} + + public format(entry: LogEntry): string { + const timeOut = this.formatTimePretty(entry.time, this.utc, this.colors) + const levelOut = this.formatLevelPretty(entry.level, this.colors) + const serviceOut = this.formatServicePretty(entry.service, this.colors) + const messageOut = entry.message ? ` ${entry.message}` : '' + const paramsOut = this.formatParametersPretty( + this.sanitize( + entry.resolvedError + ? { ...entry.resolvedError, ...entry.parameters } + : entry.parameters ?? {}, + ), + this.colors, + ) + + return `${timeOut} ${levelOut}${serviceOut}${messageOut}${paramsOut}` + } + + private formatLevelPretty(level: LogLevel, colors: boolean): string { + if (colors) { + switch (level) { + case 'CRITICAL': + case 'ERROR': + return chalk.red(chalk.bold(level.toUpperCase())) + case 'WARN': + return chalk.yellow(chalk.bold(level.toUpperCase())) + case 'INFO': + return chalk.green(chalk.bold(level.toUpperCase())) + case 'DEBUG': + return chalk.magenta(chalk.bold(level.toUpperCase())) + case 'TRACE': + return chalk.gray(chalk.bold(level.toUpperCase())) + } + } + return level.toUpperCase() + } + + private formatTimePretty(now: Date, utc: boolean, colors: boolean): string { + const h = (utc ? now.getUTCHours() : now.getHours()) + .toString() + .padStart(2, '0') + const m = (utc ? now.getUTCMinutes() : now.getMinutes()) + .toString() + .padStart(2, '0') + const s = (utc ? now.getUTCSeconds() : now.getSeconds()) + .toString() + .padStart(2, '0') + const ms = (utc ? now.getUTCMilliseconds() : now.getMilliseconds()) + .toString() + .padStart(3, '0') + + let result = `${h}:${m}:${s}.${ms}` + if (utc) { + result += 'Z' + } + + return colors ? chalk.gray(result) : result + } + + private formatParametersPretty(parameters: object, colors: boolean): string { + const oldStyles = inspect.styles + inspect.styles = STYLES + + const inspected = inspect(parameters, { + colors, + breakLength: 80 - INDENT_SIZE, + depth: 5, + }) + + inspect.styles = oldStyles + + if (inspected === '{}') { + return '' + } + + const indented = inspected + .split('\n') + .map((x) => INDENT + x) + .join('\n') + + if (colors) { + return '\n' + chalk.gray(indented) + } + return '\n' + indented + } + + private formatServicePretty( + service: string | undefined, + colors: boolean, + ): string { + if (!service) { + return '' + } + return colors + ? ` ${chalk.gray('[')} ${chalk.yellow(service)} ${chalk.gray(']')}` + : ` [ ${service} ]` + } + + private sanitize(parameters: object): object { + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return JSON.parse(toJSON(parameters)) + } +} diff --git a/packages/backend-tools/src/logger/Logger.test.ts b/packages/backend-tools/src/logger/Logger.test.ts index 0e8c41ff..c9e1f8d4 100644 --- a/packages/backend-tools/src/logger/Logger.test.ts +++ b/packages/backend-tools/src/logger/Logger.test.ts @@ -1,11 +1,22 @@ import { expect, formatCompact, mockFn } from 'earl' -import { LogEntry, Logger } from './Logger' +import { LogEntry } from './interfaces' +import { LogFormatterJson } from './LogFormatterJson' +import { LogFormatterPretty } from './LogFormatterPretty' +import { Logger } from './Logger' describe(Logger.name, () => { it('calls correct backend', () => { const backend = createTestBackend() - const logger = new Logger({ backend, logLevel: 'TRACE' }) + const logger = new Logger({ + backends: [ + { + backend, + formatter: new LogFormatterJson(), + }, + ], + logLevel: 'TRACE', + }) logger.trace('foo') logger.debug('foo') @@ -25,9 +36,13 @@ describe(Logger.name, () => { it('supports bigint values in json output', () => { const backend = createTestBackend() const logger = new Logger({ - backend, + backends: [ + { + backend, + formatter: new LogFormatterJson(), + }, + ], logLevel: 'TRACE', - format: 'json', getTime: () => new Date(0), utc: true, }) @@ -48,9 +63,13 @@ describe(Logger.name, () => { it('supports bigint values in pretty output', () => { const backend = createTestBackend() const logger = new Logger({ - backend, + backends: [ + { + backend, + formatter: new LogFormatterPretty(false, true), + }, + ], logLevel: 'TRACE', - format: 'pretty', getTime: () => new Date(0), utc: true, }) @@ -68,9 +87,13 @@ describe(Logger.name, () => { function setup() { const backend = createTestBackend() const baseLogger = new Logger({ - backend, + backends: [ + { + backend, + formatter: new LogFormatterPretty(false, true), + }, + ], logLevel: 'TRACE', - format: 'pretty', getTime: () => new Date(0), utc: true, }) diff --git a/packages/backend-tools/src/logger/Logger.ts b/packages/backend-tools/src/logger/Logger.ts index 0d0e666c..c89cd04d 100644 --- a/packages/backend-tools/src/logger/Logger.ts +++ b/packages/backend-tools/src/logger/Logger.ts @@ -2,46 +2,15 @@ import { join } from 'path' import { assertUnreachable } from '../utils/assertUnreachable' -import { formatLevelPretty } from './formatLevelPretty' -import { formatParametersPretty } from './formatParametersPretty' -import { formatServicePretty } from './formatServicePretty' -import { formatTimePretty } from './formatTimePretty' +import { LogEntry, LoggerOptions } from './interfaces' +import { LogFormatterJson } from './LogFormatterJson' +import { LogFormatterPretty } from './LogFormatterPretty' import { LEVEL, LogLevel } from './LogLevel' import { LogThrottle, LogThrottleOptions } from './LogThrottle' import { parseLogArguments } from './parseLogArguments' -import { ResolvedError, resolveError } from './resolveError' +import { resolveError } from './resolveError' import { tagService } from './tagService' -export interface LoggerBackend { - debug(message: string): void - log(message: string): void - warn(message: string): void - error(message: string): void -} - -export interface LoggerOptions { - logLevel: LogLevel - service?: string - tag?: string - format: 'pretty' | 'json' - utc: boolean - colors: boolean - cwd: string - getTime: () => Date - reportError: (entry: LogEntry) => void - backend: LoggerBackend -} - -export interface LogEntry { - level: LogLevel - time: Date - service?: string - message?: string - error?: Error - resolvedError?: ResolvedError - parameters?: object -} - /** * [Read full documentation](https://github.com/l2beat/tools/blob/master/packages/backend-tools/src/logger/docs.md) */ @@ -56,40 +25,81 @@ export class Logger { logLevel: options.logLevel ?? 'INFO', service: options.service, tag: options.tag, - format: options.format ?? 'json', utc: options.utc ?? false, - colors: options.colors ?? false, cwd: options.cwd ?? process.cwd(), getTime: options.getTime ?? (() => new Date()), reportError: options.reportError ?? (() => {}), - backend: options.backend ?? console, + backends: options.backends ?? [ + { + backend: console, + formatter: new LogFormatterJson(), + }, + ], } this.cwd = join(this.options.cwd, '/') this.logLevel = LEVEL[this.options.logLevel] } - static SILENT = new Logger({ logLevel: 'NONE', format: 'pretty' }) + static SILENT = new Logger({ logLevel: 'NONE' }) + static CRITICAL = new Logger({ logLevel: 'CRITICAL', - format: 'pretty', - colors: true, + backends: [ + { + backend: console, + formatter: new LogFormatterPretty(true, false), + }, + ], }) + static ERROR = new Logger({ logLevel: 'ERROR', - format: 'pretty', - colors: true, + backends: [ + { + backend: console, + formatter: new LogFormatterPretty(true, false), + }, + ], + }) + + static WARN = new Logger({ + logLevel: 'WARN', + backends: [ + { + backend: console, + formatter: new LogFormatterPretty(true, false), + }, + ], }) - static WARN = new Logger({ logLevel: 'WARN', format: 'pretty', colors: true }) - static INFO = new Logger({ logLevel: 'INFO', format: 'pretty', colors: true }) + + static INFO = new Logger({ + logLevel: 'INFO', + backends: [ + { + backend: console, + formatter: new LogFormatterPretty(true, false), + }, + ], + }) + static DEBUG = new Logger({ logLevel: 'DEBUG', - format: 'pretty', - colors: true, + backends: [ + { + backend: console, + formatter: new LogFormatterPretty(true, false), + }, + ], }) + static TRACE = new Logger({ logLevel: 'TRACE', - format: 'pretty', - colors: true, + backends: [ + { + backend: console, + formatter: new LogFormatterPretty(true, false), + }, + ], }) configure(options: Partial): Logger { @@ -189,78 +199,28 @@ export class Logger { } private printExactly(entry: LogEntry): void { - const output = - this.options.format === 'json' - ? this.formatJson(entry) - : this.formatPretty(entry) - - switch (entry.level) { - case 'CRITICAL': - case 'ERROR': - this.options.backend.error(output) - break - case 'WARN': - this.options.backend.warn(output) - break - case 'INFO': - this.options.backend.log(output) - break - case 'DEBUG': - case 'TRACE': - this.options.backend.debug(output) - break - case 'NONE': - break - default: - assertUnreachable(entry.level) - } - } - - private formatJson(entry: LogEntry): string { - const core = { - time: entry.time.toISOString(), - level: entry.level, - service: entry.service, - message: entry.message, - error: entry.resolvedError, - } - try { - return toJSON({ ...core, parameters: entry.parameters }) - } catch (e) { - this.error('Unable to log', e) - return JSON.stringify(core) - } - } - - private formatPretty(entry: LogEntry): string { - const timeOut = formatTimePretty( - entry.time, - this.options.utc, - this.options.colors, - ) - const levelOut = formatLevelPretty(entry.level, this.options.colors) - const serviceOut = formatServicePretty(entry.service, this.options.colors) - const messageOut = entry.message ? ` ${entry.message}` : '' - const paramsOut = formatParametersPretty( - sanitize( - entry.resolvedError - ? { ...entry.resolvedError, ...entry.parameters } - : entry.parameters ?? {}, - ), - this.options.colors, - ) - - return `${timeOut} ${levelOut}${serviceOut}${messageOut}${paramsOut}` + this.options.backends.forEach((backendOptions) => { + const output = backendOptions.formatter.format(entry) + switch (entry.level) { + case 'CRITICAL': + case 'ERROR': + backendOptions.backend.error(output) + break + case 'WARN': + backendOptions.backend.warn(output) + break + case 'INFO': + backendOptions.backend.log(output) + break + case 'DEBUG': + case 'TRACE': + backendOptions.backend.debug(output) + break + case 'NONE': + break + default: + assertUnreachable(entry.level) + } + }) } } - -function toJSON(parameters: {}): string { - return JSON.stringify(parameters, (k, v: unknown) => - typeof v === 'bigint' ? v.toString() : v, - ) -} - -function sanitize(parameters: {}): {} { - // eslint-disable-next-line @typescript-eslint/no-unsafe-return - return JSON.parse(toJSON(parameters)) -} diff --git a/packages/backend-tools/src/logger/docs.md b/packages/backend-tools/src/logger/docs.md index 8081cf11..44ecb200 100644 --- a/packages/backend-tools/src/logger/docs.md +++ b/packages/backend-tools/src/logger/docs.md @@ -26,13 +26,11 @@ The logger can be configured using the following options, all of them optional: - `logLevel` - minimum level of messages that will be logged, defaults to `INFO`. See more in the [Levels](#levels) section. - `service` - name of the service (class) that is using the logger. See more in the [Services](#services) section. - `tag` - tag that is used to identify the logger. See more in the [Tags](#tags) section. -- `format` - either `pretty` or `json`. It is recommended to use the `pretty` format during development and the `json` format in production. Defaults to `json`. - `utc` - when set to true time is logged in UTC, otherwise in local time. Defaults to `false`. -- `colors` - when set to true colors are used in the `pretty` format, otherwise they are not. Defaults to `false`. - `cwd` - current working directory, used to shorten error stack traces. Defaults to `process.cwd()`. - `getTime` - callback that returns the current time. Defaults to `() => new Date()`. - `reportError` - callback called when a message is logged at the `ERROR` or `CRITICAL` level. See more in the [Error reporting](#error-reporting) section. -- `backend` - object that is used to log messages. Defaults to `console`. +- `backends` - a set of pairs ([backend](#backends) + [formatter](#formatters)) which define where and in what form logs are being outputed. Defaults to `console` and `pretty` formatter ### Services @@ -119,9 +117,27 @@ This is done using the following rules: - non-object arguments are stored as `parameters.value` or `parameters.values` depending on the number of such arguments - object arguments are merged into a single `parameters` object +## Backends + +Currently we support two backends + +- `console` - standard output to console +- `ElasticSearchBackend` - pushes logs ElasticSearch node (should be used together with [ECS formatter](#ecs)) + +## Formatters + +Along with each backend it is required to provide a formatter which will produce an output string for each log entry + ### Pretty -In this format every message is logged on one or more lines with another newline in between the messages. The first line contains the timestamp, log level, service, tag and the message. The following lines contain a representation of the parameters. +Type: `LogFormarretPretty` + +In this format every message is logged on one or more lines with another newline in between the messages. The first line contains the timestamp, log level, service, tag and the message. The following lines contain a representation of the parameters. This form is best suited for local development purposes. + +This formatter accepts two params: + +- `utc` - when set to true time is logged in UTC, otherwise in local time. Defaults to `false`. +- `colors` - when set to true colors are used in the `pretty` format, otherwise they are not. Defaults to `false`. Below is an example log output: @@ -142,7 +158,9 @@ Below is an example log output: ### JSON -In this format every message is logged on a single line as a single JSON object. The object contains the timestamp, log level, service, tag, message, error and parameters. +Type: `LogFormarretJson` + +In this format every message is logged on a single line as a single JSON object. The object contains the timestamp, log level, service, tag, message, error and parameters. This format is best suited for deploment environments. Below is an example log output: @@ -151,6 +169,20 @@ Below is an example log output: {"time":"2023-01-02T12:34:56.002Z","level":"ERROR","service":"PriceService:USD","message":"Error fetching prices","error":{"name":"Error","error":"429: You have been rate limited!","stack":["PriceService.fetchPrices (src/PriceService.ts:12:34)","TaxService.computeTaxes (src/TaxService.ts:56:78)"]}} ``` +### ECS + +Type: `LogFormarretEcs` + +In this format every message is logged on a single line as a single JSON object compatible with [ECS standard](https://www.elastic.co/guide/en/ecs/current/ecs-reference.html). This format is best suited for deploment environments with ElastiSearch enabled + +Below is an example log output: + +``` +{"@timestamp":"2024-04-25T15:47:52.731Z","log":{"level":"INFO"},"service":{"name":"Application"},"message":"Log level","parameters":{"value":"INFO"}} +{"@timestamp":"2024-04-25T15:47:52.733Z","log":{"level":"INFO"},"service":{"name":"ApiServer"},"message":"Listening","parameters":{"port":3000}} +{"@timestamp":"2024-04-25T15:47:52.864Z","log":{"level":"INFO"},"service":{"name":"Database"},"message":"Migrations completed","parameters":{"version":"105"}} +``` + ## Error reporting It might be useful to connect the logger into an error reporting system. This can be done by providing a `reportError` callback that will be called when a message is logged at the `ERROR` or `CRITICAL` level. diff --git a/packages/backend-tools/src/logger/formatLevelPretty.ts b/packages/backend-tools/src/logger/formatLevelPretty.ts deleted file mode 100644 index 50e4ea24..00000000 --- a/packages/backend-tools/src/logger/formatLevelPretty.ts +++ /dev/null @@ -1,22 +0,0 @@ -import chalk from 'chalk' - -import { LogLevel } from './LogLevel' - -export function formatLevelPretty(level: LogLevel, colors: boolean): string { - if (colors) { - switch (level) { - case 'CRITICAL': - case 'ERROR': - return chalk.red(chalk.bold(level.toUpperCase())) - case 'WARN': - return chalk.yellow(chalk.bold(level.toUpperCase())) - case 'INFO': - return chalk.green(chalk.bold(level.toUpperCase())) - case 'DEBUG': - return chalk.magenta(chalk.bold(level.toUpperCase())) - case 'TRACE': - return chalk.gray(chalk.bold(level.toUpperCase())) - } - } - return level.toUpperCase() -} diff --git a/packages/backend-tools/src/logger/formatParametersPretty.ts b/packages/backend-tools/src/logger/formatParametersPretty.ts deleted file mode 100644 index 64d2af76..00000000 --- a/packages/backend-tools/src/logger/formatParametersPretty.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* eslint-disable @typescript-eslint/ban-types */ -import chalk from 'chalk' -import { inspect } from 'util' - -const STYLES = { - bigint: 'white', - boolean: 'white', - date: 'white', - module: 'white', - name: 'blue', - null: 'white', - number: 'white', - regexp: 'white', - special: 'white', - string: 'white', - symbol: 'white', - undefined: 'white', -} - -const INDENT_SIZE = 4 -const INDENT = ' '.repeat(INDENT_SIZE) - -export function formatParametersPretty( - parameters: {}, - colors: boolean, -): string { - const oldStyles = inspect.styles - inspect.styles = STYLES - - const inspected = inspect(parameters, { - colors, - breakLength: 80 - INDENT_SIZE, - depth: 5, - }) - - inspect.styles = oldStyles - - if (inspected === '{}') { - return '' - } - - const indented = inspected - .split('\n') - .map((x) => INDENT + x) - .join('\n') - - if (colors) { - return '\n' + chalk.gray(indented) - } - return '\n' + indented -} diff --git a/packages/backend-tools/src/logger/formatServicePretty.ts b/packages/backend-tools/src/logger/formatServicePretty.ts deleted file mode 100644 index ede9d362..00000000 --- a/packages/backend-tools/src/logger/formatServicePretty.ts +++ /dev/null @@ -1,13 +0,0 @@ -import chalk from 'chalk' - -export function formatServicePretty( - service: string | undefined, - colors: boolean, -): string { - if (!service) { - return '' - } - return colors - ? ` ${chalk.gray('[')} ${chalk.yellow(service)} ${chalk.gray(']')}` - : ` [ ${service} ]` -} diff --git a/packages/backend-tools/src/logger/formatTimePretty.ts b/packages/backend-tools/src/logger/formatTimePretty.ts deleted file mode 100644 index fa065ca7..00000000 --- a/packages/backend-tools/src/logger/formatTimePretty.ts +++ /dev/null @@ -1,27 +0,0 @@ -import chalk from 'chalk' - -export function formatTimePretty( - now: Date, - utc: boolean, - colors: boolean, -): string { - const h = (utc ? now.getUTCHours() : now.getHours()) - .toString() - .padStart(2, '0') - const m = (utc ? now.getUTCMinutes() : now.getMinutes()) - .toString() - .padStart(2, '0') - const s = (utc ? now.getUTCSeconds() : now.getSeconds()) - .toString() - .padStart(2, '0') - const ms = (utc ? now.getUTCMilliseconds() : now.getMilliseconds()) - .toString() - .padStart(3, '0') - - let result = `${h}:${m}:${s}.${ms}` - if (utc) { - result += 'Z' - } - - return colors ? chalk.gray(result) : result -} diff --git a/packages/backend-tools/src/logger/interfaces.ts b/packages/backend-tools/src/logger/interfaces.ts new file mode 100644 index 00000000..d8404124 --- /dev/null +++ b/packages/backend-tools/src/logger/interfaces.ts @@ -0,0 +1,39 @@ +import { LogLevel } from './LogLevel' +import { ResolvedError } from './resolveError' + +export interface LoggerBackend { + debug(message: string): void + log(message: string): void + warn(message: string): void + error(message: string): void +} + +export interface LogFormatter { + format(entry: LogEntry): string +} + +export interface LoggerBackendOptions { + backend: LoggerBackend + formatter: LogFormatter +} + +export interface LoggerOptions { + logLevel: LogLevel + service?: string + tag?: string + utc: boolean + cwd: string + getTime: () => Date + reportError: (entry: LogEntry) => void + backends: LoggerBackendOptions[] +} + +export interface LogEntry { + level: LogLevel + time: Date + service?: string + message?: string + error?: Error + resolvedError?: ResolvedError + parameters?: object +} diff --git a/packages/backend-tools/src/logger/toJSON.ts b/packages/backend-tools/src/logger/toJSON.ts new file mode 100644 index 00000000..45b74ff2 --- /dev/null +++ b/packages/backend-tools/src/logger/toJSON.ts @@ -0,0 +1,5 @@ +export function toJSON(parameters: object): string { + return JSON.stringify(parameters, (k, v: unknown) => + typeof v === 'bigint' ? v.toString() : v, + ) +} diff --git a/packages/discovery/CHANGELOG.md b/packages/discovery/CHANGELOG.md index 1c604442..b85a4642 100644 --- a/packages/discovery/CHANGELOG.md +++ b/packages/discovery/CHANGELOG.md @@ -1,5 +1,12 @@ # @l2beat/discovery +## 0.47.2 + +### Patch Changes + +- Updated dependencies + - @l2beat/backend-tools@0.6.0 + ## 0.47.1 ### Patch Changes diff --git a/packages/discovery/package.json b/packages/discovery/package.json index b349b3cc..bcde1321 100644 --- a/packages/discovery/package.json +++ b/packages/discovery/package.json @@ -1,7 +1,7 @@ { "name": "@l2beat/discovery", "description": "L2Beat discovery - engine & tooling utilized for keeping an eye on L2s", - "version": "0.47.1", + "version": "0.47.2", "main": "dist/index.js", "types": "dist/index.d.ts", "bin": { @@ -19,7 +19,7 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@l2beat/backend-tools": "^0.5.1", + "@l2beat/backend-tools": "^0.6.0", "@l2beat/discovery-types": "^0.8.1", "@mradomski/fast-solidity-parser": "0.1.1", "chalk": "^4.1.2", diff --git a/packages/uif/CHANGELOG.md b/packages/uif/CHANGELOG.md index 1fe3ae26..f88b57f9 100644 --- a/packages/uif/CHANGELOG.md +++ b/packages/uif/CHANGELOG.md @@ -1,5 +1,12 @@ # @l2beat/uif +## 0.5.1 + +### Patch Changes + +- Updated dependencies + - @l2beat/backend-tools@0.6.0 + ## 0.5.0 ### Minor Changes diff --git a/packages/uif/package.json b/packages/uif/package.json index 58a891bc..5c228262 100644 --- a/packages/uif/package.json +++ b/packages/uif/package.json @@ -1,7 +1,7 @@ { "name": "@l2beat/uif", "description": "Universal Indexer Framework.", - "version": "0.5.0", + "version": "0.5.1", "license": "MIT", "repository": "https://github.com/l2beat/tools", "bugs": { @@ -30,7 +30,7 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@l2beat/backend-tools": "^0.5.0" + "@l2beat/backend-tools": "^0.6.0" }, "devDependencies": { "@sinonjs/fake-timers": "^11.1.0", diff --git a/yarn.lock b/yarn.lock index 724af882..dc40f5dc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -237,6 +237,26 @@ dependencies: "@jridgewell/trace-mapping" "0.3.9" +"@elastic/elasticsearch@^8.13.1": + version "8.13.1" + resolved "https://registry.yarnpkg.com/@elastic/elasticsearch/-/elasticsearch-8.13.1.tgz#0fbe8318cf7f21c599165bb901277428639d57ec" + integrity sha512-2G4Vu6OHw4+XTrp7AGIcOEezpPEoVrWg2JTK1v/exEKSLYquZkUdd+m4yOL3/UZ6bTj7hmXwrmYzW76BnLCkJQ== + dependencies: + "@elastic/transport" "~8.4.1" + tslib "^2.4.0" + +"@elastic/transport@~8.4.1": + version "8.4.1" + resolved "https://registry.yarnpkg.com/@elastic/transport/-/transport-8.4.1.tgz#f98c5a5e2156bcb3f01170b4aca7e7de4d8b61b8" + integrity sha512-/SXVuVnuU5b4dq8OFY4izG+dmGla185PcoqgK6+AJMpmOeY1QYVNbWtCwvSvoAANN5D/wV+EBU8+x7Vf9EphbA== + dependencies: + debug "^4.3.4" + hpagent "^1.0.0" + ms "^2.1.3" + secure-json-parse "^2.4.0" + tslib "^2.4.0" + undici "^5.22.1" + "@esbuild/android-arm64@0.17.19": version "0.17.19" resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.17.19.tgz#bafb75234a5d3d1b690e7c2956a599345e84a2fd" @@ -721,6 +741,11 @@ "@ethersproject/properties" "^5.7.0" "@ethersproject/strings" "^5.7.0" +"@fastify/busboy@^2.0.0": + version "2.1.1" + resolved "https://registry.yarnpkg.com/@fastify/busboy/-/busboy-2.1.1.tgz#b9da6a878a371829a0502c9b6c1c143ef6663f4d" + integrity sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA== + "@gar/promisify@^1.0.1": version "1.1.3" resolved "https://registry.yarnpkg.com/@gar/promisify/-/promisify-1.1.3.tgz#555193ab2e3bb3b6adc3d551c9c030d9e860daf6" @@ -912,6 +937,11 @@ resolved "https://registry.yarnpkg.com/@types/deep-diff/-/deep-diff-1.0.2.tgz#36f1291f0aead8aceb847cde6f07ae613a78ac4f" integrity sha512-WD2O611C7Oz7RSwKbSls8LaznKfWfXh39CHY9Amd8FhQz+NJRe20nUHhYpOopVq9M2oqDZd4L6AzqJIXQycxiA== +"@types/elasticsearch@^5.0.43": + version "5.0.43" + resolved "https://registry.yarnpkg.com/@types/elasticsearch/-/elasticsearch-5.0.43.tgz#a3bbf56922de2d0e24c6117e8de1c9b50029c3c6" + integrity sha512-N+MpzURpDCWd7zaJ7CE1aU+nBSeAABLhDE0lGodQ0LLftx7ku6hjTXLr9OAFZLSXiWL3Xxx8jts485ynrcm5NA== + "@types/is-ci@^3.0.0": version "3.0.0" resolved "https://registry.yarnpkg.com/@types/is-ci/-/is-ci-3.0.0.tgz#7e8910af6857601315592436f030aaa3ed9783c3" @@ -1002,6 +1032,11 @@ resolved "https://registry.yarnpkg.com/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-8.1.2.tgz#bf2e02a3dbd4aecaf95942ecd99b7402e03fad5e" integrity sha512-9GcLXF0/v3t80caGs5p2rRfkB+a8VBGLJZVih6CNFkx8IZ994wiKKLSRs9nuFwk1HevWs/1mnUmkApGrSGsShA== +"@types/uuid@^9.0.8": + version "9.0.8" + resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-9.0.8.tgz#7545ba4fc3c003d6c756f651f3bf163d8f0f29ba" + integrity sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA== + "@types/yargs-parser@*": version "21.0.0" resolved "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.0.tgz" @@ -2623,6 +2658,11 @@ hosted-git-info@^2.1.4: resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.9.tgz#dffc0bf9a21c02209090f2aa69429e1414daf3f9" integrity sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw== +hpagent@^1.0.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/hpagent/-/hpagent-1.2.0.tgz#0ae417895430eb3770c03443456b8d90ca464903" + integrity sha512-A91dYTeIB6NoXG+PxTQpCCDDnfHsW9kc06Lvpu1TEe9gnd6ZFeiBoRO9JvzEv6xK7EX97/dUE8g/vBMTqTS3CA== + http-cache-semantics@^4.1.0: version "4.1.1" resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a" @@ -3358,7 +3398,7 @@ ms@2.1.2: resolved "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -ms@2.1.3, ms@^2.0.0, ms@^2.1.1: +ms@2.1.3, ms@^2.0.0, ms@^2.1.1, ms@^2.1.3: version "2.1.3" resolved "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== @@ -3942,6 +3982,11 @@ scrypt-js@3.0.1: resolved "https://registry.yarnpkg.com/scrypt-js/-/scrypt-js-3.0.1.tgz#d314a57c2aef69d1ad98a138a21fe9eafa9ee312" integrity sha512-cdwTTnqPu0Hyvf5in5asVdZocVDTNRmR7XEcJuIzMjJeSHybHl7vpB66AzwTaIg6CLSbtjcxc8fqcySfnTkccA== +secure-json-parse@^2.4.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/secure-json-parse/-/secure-json-parse-2.7.0.tgz#5a5f9cd6ae47df23dba3151edd06855d47e09862" + integrity sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw== + "semver@2 || 3 || 4 || 5": version "5.7.2" resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8" @@ -4396,6 +4441,11 @@ tslib@^1.8.1: resolved "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz" integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== +tslib@^2.4.0: + version "2.6.2" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" + integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== + tsutils@^3.21.0: version "3.21.0" resolved "https://registry.npmjs.org/tsutils/-/tsutils-3.21.0.tgz" @@ -4472,6 +4522,13 @@ unbox-primitive@^1.0.2: has-symbols "^1.0.3" which-boxed-primitive "^1.0.2" +undici@^5.22.1: + version "5.28.4" + resolved "https://registry.yarnpkg.com/undici/-/undici-5.28.4.tgz#6b280408edb6a1a604a9b20340f45b422e373068" + integrity sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g== + dependencies: + "@fastify/busboy" "^2.0.0" + unique-filename@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/unique-filename/-/unique-filename-1.1.1.tgz#1d69769369ada0583103a1e6ae87681b56573230" @@ -4503,6 +4560,11 @@ util-deprecate@^1.0.1: resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw== +uuid@^9.0.1: + version "9.0.1" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.1.tgz#e188d4c8853cc722220392c424cd637f32293f30" + integrity sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA== + v8-compile-cache-lib@^3.0.1: version "3.0.1" resolved "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz"