From 6dcafe206a16f363a6218b547dabb1e2cefda7c1 Mon Sep 17 00:00:00 2001 From: Duyet Le Date: Wed, 25 Sep 2024 12:18:04 +0700 Subject: [PATCH 1/3] feat: add charts to failed-query page --- app/[host]/[query]/queries/failed-queries.ts | 127 ++++++++++++------ .../charts/failed-query-count-by-user.tsx | 71 ++++++++++ components/charts/failed-query-count.tsx | 91 +++++++++++++ 3 files changed, 247 insertions(+), 42 deletions(-) create mode 100644 components/charts/failed-query-count-by-user.tsx create mode 100644 components/charts/failed-query-count.tsx diff --git a/app/[host]/[query]/queries/failed-queries.ts b/app/[host]/[query]/queries/failed-queries.ts index 93b3e729..b6700dad 100644 --- a/app/[host]/[query]/queries/failed-queries.ts +++ b/app/[host]/[query]/queries/failed-queries.ts @@ -3,45 +3,46 @@ import { type QueryConfig } from '@/types/query-config' export const failedQueriesConfig: QueryConfig = { name: 'failed-queries', + description: "type IN ['ExceptionBeforeStart', 'ExceptionWhileProcessing']", sql: ` - SELECT - type, - query_start_time, - query_duration_ms, - query_id, - query_kind, - is_initial_query, - normalizeQuery(query) AS normalized_query, - concat(toString(read_rows), ' rows / ', formatReadableSize(read_bytes)) AS read, - concat(toString(written_rows), ' rows / ', formatReadableSize(written_bytes)) AS written, - concat(toString(result_rows), ' rows / ', formatReadableSize(result_bytes)) AS result, - formatReadableSize(memory_usage) AS memory_usage, - exception, - concat('\n', stack_trace) AS stack_trace, - user, - initial_user, - multiIf(empty(client_name), http_user_agent, concat(client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch))) AS client, - client_hostname, - databases, - tables, - columns, - used_aggregate_functions, - used_aggregate_function_combinators, - used_database_engines, - used_data_type_families, - used_dictionaries, - used_formats, - used_functions, - used_storages, - used_table_functions, - thread_ids, - ProfileEvents, - Settings - FROM system.query_log - WHERE type IN ['ExceptionBeforeStart', 'ExceptionWhileProcessing'] - ORDER BY query_start_time DESC - LIMIT 100 - `, + SELECT + type, + query_start_time, + query_duration_ms, + query_id, + query_kind, + is_initial_query, + normalizeQuery(query) AS normalized_query, + concat(toString(read_rows), ' rows / ', formatReadableSize(read_bytes)) AS read, + concat(toString(written_rows), ' rows / ', formatReadableSize(written_bytes)) AS written, + concat(toString(result_rows), ' rows / ', formatReadableSize(result_bytes)) AS result, + formatReadableSize(memory_usage) AS memory_usage, + exception, + concat('\n', stack_trace) AS stack_trace, + user, + initial_user, + multiIf(empty(client_name), http_user_agent, concat(client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch))) AS client, + client_hostname, + toString(databases) AS databases, + toString(tables) AS tables, + toString(columns) AS columns, + toString(used_aggregate_functions) AS used_aggregate_functions, + toString(used_aggregate_function_combinators) AS used_aggregate_function_combinators, + toString(used_database_engines) AS used_database_engines, + toString(used_data_type_families) AS used_data_type_families, + toString(used_dictionaries) AS used_dictionaries, + toString(used_formats) AS used_formats, + toString(used_functions) AS used_functions, + toString(used_storages) AS used_storages, + toString(used_table_functions) AS used_table_functions, + toString(thread_ids) AS thread_ids, + ProfileEvents, + Settings + FROM system.query_log + WHERE type IN ['ExceptionBeforeStart', 'ExceptionWhileProcessing'] + ORDER BY query_start_time DESC + LIMIT 1000 + `, columns: [ 'normalized_query', 'exception', @@ -75,14 +76,56 @@ export const failedQueriesConfig: QueryConfig = { 'thread_ids', ], columnFormats: { - normalized_query: ColumnFormat.Code, + normalized_query: [ + ColumnFormat.CodeDialog, + { + trigger_classname: 'w-80 line-clamp-4', + dialog_classname: 'max-w-screen-xl', + max_truncate: 200, + }, + ], type: ColumnFormat.ColoredBadge, query_duration_ms: ColumnFormat.Duration, query_start_time: ColumnFormat.RelatedTime, - exception: ColumnFormat.CodeDialog, - stack_trace: ColumnFormat.CodeToggle, - client: ColumnFormat.Code, + exception: [ + ColumnFormat.CodeDialog, + { trigger_classname: 'w-80 line-clamp-2' }, + ], + stack_trace: ColumnFormat.CodeDialog, + client: ColumnFormat.CodeDialog, + user: ColumnFormat.ColoredBadge, + initial_user: ColumnFormat.ColoredBadge, is_initial_query: ColumnFormat.Boolean, query_kind: ColumnFormat.Badge, + databases: ColumnFormat.CodeDialog, + tables: ColumnFormat.CodeDialog, + columns: ColumnFormat.CodeDialog, + used_aggregate_functions: ColumnFormat.CodeDialog, + used_aggregate_function_combinators: ColumnFormat.CodeDialog, + used_formats: ColumnFormat.CodeDialog, + used_dictionaries: ColumnFormat.CodeDialog, + used_functions: ColumnFormat.CodeDialog, + used_table_functions: ColumnFormat.CodeDialog, + thread_ids: ColumnFormat.CodeDialog, }, + relatedCharts: [ + [ + 'failed-query-count', + { + title: 'Failed Queries over last 14 days', + interval: 'toStartOfHour', + lastHours: 24 * 14, + showLegend: false, + }, + ], + [ + 'failed-query-count-by-user', + { + title: 'Failed Queries over last 14 days by Users', + interval: 'toStartOfHour', + lastHours: 24 * 14, + showLegend: false, + }, + ], + ], } diff --git a/components/charts/failed-query-count-by-user.tsx b/components/charts/failed-query-count-by-user.tsx new file mode 100644 index 00000000..f5cc9f4a --- /dev/null +++ b/components/charts/failed-query-count-by-user.tsx @@ -0,0 +1,71 @@ +import { type ChartProps } from '@/components/charts/chart-props' +import { BarChart } from '@/components/generic-charts/bar' +import { ChartCard } from '@/components/generic-charts/chart-card' +import { fetchData } from '@/lib/clickhouse' +import { applyInterval } from '@/lib/clickhouse-query' + +export async function ChartFailedQueryCountByType({ + title, + interval = 'toStartOfDay', + lastHours = 24 * 14, + className, + chartClassName, + ...props +}: ChartProps) { + const query = ` + SELECT ${applyInterval(interval, 'event_time')}, + user, + countDistinct(query_id) AS count + FROM merge(system, '^query_log') + WHERE + type IN ['ExceptionBeforeStart', 'ExceptionWhileProcessing'] + AND event_time >= (now() - INTERVAL ${lastHours} HOUR) + GROUP BY 1, 2 + ORDER BY + 1 ASC, + 3 DESC + ` + const { data: raw } = await fetchData< + { + event_time: string + user: string + count: number + }[] + >({ query }) + + const data = raw.reduce( + (acc, cur) => { + const { event_time, user, count } = cur + if (acc[event_time] === undefined) { + acc[event_time] = {} + } + acc[event_time][user] = count + return acc + }, + {} as Record> + ) + + const barData = Object.entries(data).map(([event_time, obj]) => { + return { event_time, ...obj } + }) + + const users = Object.values(data).reduce((acc, cur) => { + return Array.from(new Set([...acc, ...Object.keys(cur)])) + }, [] as string[]) + + return ( + + + + ) +} + +export default ChartFailedQueryCountByType diff --git a/components/charts/failed-query-count.tsx b/components/charts/failed-query-count.tsx new file mode 100644 index 00000000..8ee2670a --- /dev/null +++ b/components/charts/failed-query-count.tsx @@ -0,0 +1,91 @@ +import { type ChartProps } from '@/components/charts/chart-props' +import { AreaChart } from '@/components/generic-charts/area' +import { ChartCard } from '@/components/generic-charts/chart-card' +import { fetchData } from '@/lib/clickhouse' +import { cn } from '@/lib/utils' + +export async function ChartFailedQueryCount({ + title, + interval = 'toStartOfMinute', + className, + chartClassName, + chartCardContentClassName, + lastHours = 24, + showXAxis = true, + showLegend = false, + showCartesianGrid = true, + breakdown = 'breakdown', + ...props +}: ChartProps) { + const query = ` + WITH event_count AS ( + SELECT ${interval}(event_time) AS event_time, + COUNT() AS query_count + FROM merge(system, '^query_log') + WHERE + type IN ['ExceptionBeforeStart', 'ExceptionWhileProcessing'] + AND event_time >= (now() - INTERVAL ${lastHours} HOUR) + GROUP BY 1 + ORDER BY 1 + ), + query_type AS ( + SELECT ${interval}(event_time) AS event_time, + type AS query_type, + COUNT() AS count + FROM merge(system, '^query_log') + WHERE + type IN ['ExceptionBeforeStart', 'ExceptionWhileProcessing'] + AND event_time >= (now() - INTERVAL ${lastHours} HOUR) + GROUP BY 1, 2 + ORDER BY 3 DESC + ), + breakdown AS ( + SELECT event_time, + groupArray((query_type, count)) AS breakdown + FROM query_type + GROUP BY 1 + ) + SELECT event_time, + query_count, + breakdown.breakdown AS breakdown + FROM event_count + LEFT JOIN breakdown USING event_time + ORDER BY 1 + ` + const { data } = await fetchData< + { + event_time: string + query_count: number + breakdown: Array<[string, number] | Record> + }[] + >({ query }) + + return ( + + + + ) +} + +export default ChartFailedQueryCount From 0cdd1dec314ff3d3500e43bfccfa25035e5b7e1e Mon Sep 17 00:00:00 2001 From: Duyet Le Date: Wed, 25 Sep 2024 12:18:17 +0700 Subject: [PATCH 2/3] fix(test): update code-dialog-format test --- components/data-table/cells/code-dialog-format.cy.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/data-table/cells/code-dialog-format.cy.tsx b/components/data-table/cells/code-dialog-format.cy.tsx index 42c24926..dae2d04b 100644 --- a/components/data-table/cells/code-dialog-format.cy.tsx +++ b/components/data-table/cells/code-dialog-format.cy.tsx @@ -5,7 +5,7 @@ describe('', () => { const shortCode = 'SELECT * FROM table' cy.mount() cy.get('code').should('contain.text', shortCode) - cy.get('button svg[role="open-dialog"]').should('not.exist') + cy.get('svg[role="open-dialog"]').should('not.exist') }) it('renders long code with dialog', () => { @@ -13,14 +13,14 @@ describe('', () => { 'SELECT * FROM table WHERE column1 = "value" AND column2 > 100 ORDER BY column3 DESC LIMIT 10' cy.mount() cy.get('code').should('exist') - cy.get('button svg[role="open-dialog"]').should('exist') + cy.get('svg[role="open-dialog"]').should('exist') }) it('opens dialog on button click', () => { const longCode = 'SELECT * FROM table WHERE column1 = "value" AND column2 > 100 ORDER BY column3 DESC LIMIT 10' cy.mount() - cy.get('button svg[role="open-dialog"]').click() + cy.get('svg[role="open-dialog"]').click() cy.get('div[role="dialog"]').should('be.visible') cy.get('div[role="dialog"] code').should('contain.text', longCode) }) @@ -35,7 +35,7 @@ describe('', () => { hide_query_comment: true, } cy.mount() - cy.get('button svg[role="open-dialog"]').click() + cy.get('svg[role="open-dialog"]').click() cy.get('div[role="dialog"]').within(() => { cy.contains('Custom Title').should('be.visible') cy.contains('Custom Description').should('be.visible') From 4db2111991b6a3ab772fb09ccd854dc35c5a97e0 Mon Sep 17 00:00:00 2001 From: Duyet Le Date: Wed, 25 Sep 2024 12:27:33 +0700 Subject: [PATCH 3/3] fix(test): update jest test --- lib/clickhouse-query.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/clickhouse-query.test.ts b/lib/clickhouse-query.test.ts index eb0244a8..197be5ee 100644 --- a/lib/clickhouse-query.test.ts +++ b/lib/clickhouse-query.test.ts @@ -4,21 +4,21 @@ import { applyInterval } from './clickhouse-query' describe('applyInterval', () => { it('should apply toStartOfDay for toStartOfDay interval', () => { const result = applyInterval('toStartOfDay', 'myColumn', 'myAlias') - expect(result).toEqual('toDate(toStartOfDay(myColumn)) as myAlias') // Change to toEqual + expect(result).toEqual('toDate(toStartOfDay(myColumn)) AS myAlias') // Change to toEqual }) it('should apply toStartOfWeek for toStartOfWeek interval', () => { const result = applyInterval('toStartOfWeek', 'myColumn') - expect(result).toEqual('toDate(toStartOfWeek(myColumn)) as myColumn') // Change to toEqual + expect(result).toEqual('toDate(toStartOfWeek(myColumn)) AS myColumn') // Change to toEqual }) it('should apply toStartOfMonth for toStartOfMonth interval', () => { const result = applyInterval('toStartOfMonth', 'myColumn', 'myAlias') - expect(result).toEqual('toDate(toStartOfMonth(myColumn)) as myAlias') // Change to toEqual + expect(result).toEqual('toDate(toStartOfMonth(myColumn)) AS myAlias') // Change to toEqual }) it('should apply toStartOfHour for other intervals', () => { const result = applyInterval('toStartOfHour', 'myColumn', 'myAlias') - expect(result).toEqual('toStartOfHour(myColumn) as myAlias') // Change to toEqual + expect(result).toEqual('toStartOfHour(myColumn) AS myAlias') // Change to toEqual }) })