From 27ac387807f3d6da523de8563b63bc9b1562616f Mon Sep 17 00:00:00 2001 From: Duyet Le Date: Mon, 3 Jun 2024 16:36:53 +0700 Subject: [PATCH] feat: add `CLICKHOUSE_MAX_EXECUTION_TIME`, update fetchData --- .env.example | 1 + README.md | 5 ++- app/[query]/table.tsx | 7 +++- app/api/timezone/route.ts | 5 ++- app/api/version/route.ts | 5 ++- app/clusters/[cluster]/breadcrumb.tsx | 10 ++--- .../[cluster]/count-across-replicas/page.tsx | 6 +-- .../[cluster]/parts-across-replicas/page.tsx | 6 +-- .../[cluster]/replicas-status/page.tsx | 4 +- app/clusters/page.tsx | 4 +- app/dashboard/render-chart.tsx | 2 +- app/dashboard/seeding.ts | 4 +- app/dashboard/utils.ts | 12 +++--- .../[database]/[table]/@mergetree/page.tsx | 42 +++++++++++-------- .../[database]/[table]/engine-type.ts | 6 +-- .../[table]/extras/alternative-tables.tsx | 10 ++++- .../[table]/extras/running-queries-count.tsx | 15 ++++--- .../[table]/extras/running-queries.tsx | 8 ++-- .../[database]/[table]/extras/sample-data.tsx | 8 ++-- .../[database]/[table]/extras/table-ddl.tsx | 5 ++- .../[database]/[table]/extras/table-info.tsx | 5 ++- app/database/[database]/breadcrumb.tsx | 11 ++--- app/database/[database]/page.tsx | 4 +- app/explain/actions.ts | 2 +- app/replica/[replica]/tables/page.tsx | 4 +- components/charts/backup-size.tsx | 2 +- components/charts/connections-http.tsx | 2 +- components/charts/connections-interserver.tsx | 2 +- components/charts/cpu-usage.tsx | 2 +- components/charts/disk-size.tsx | 2 +- components/charts/disks-usage.tsx | 2 +- components/charts/memory-usage.tsx | 2 +- components/charts/merge-avg-duration.tsx | 2 +- components/charts/merge-count.tsx | 2 +- components/charts/merge-sum-read-rows.tsx | 2 +- components/charts/new-parts-created.tsx | 2 +- components/charts/query-count-by-user.tsx | 2 +- components/charts/query-count.tsx | 2 +- components/charts/query-duration.tsx | 2 +- components/charts/query-memory.tsx | 2 +- components/charts/query-type.tsx | 2 +- components/charts/readonly-replica.tsx | 2 +- components/charts/replication-queue-count.tsx | 4 +- .../charts/replication-summary-table.tsx | 2 +- components/charts/summary-used-by-merges.tsx | 17 ++++---- .../charts/summary-used-by-mutations.tsx | 4 +- .../summary-used-by-running-queries.tsx | 16 +++---- components/charts/top-table-size.tsx | 4 +- components/charts/zookeeper-requests.tsx | 2 +- components/charts/zookeeper-summary-table.tsx | 2 +- components/charts/zookeeper-uptime.tsx | 4 +- components/charts/zookeeper-wait.tsx | 2 +- components/clickhouse-host.tsx | 5 ++- .../data-table/cells/actions/actions.ts | 6 +-- components/data-table/data-table.tsx | 2 +- components/menu/count-badge.tsx | 26 ++++++------ lib/clickhouse.ts | 18 ++++++-- lib/types/query-config.ts | 5 +++ 58 files changed, 194 insertions(+), 150 deletions(-) diff --git a/.env.example b/.env.example index 602b1da7..37ce9209 100644 --- a/.env.example +++ b/.env.example @@ -5,3 +5,4 @@ CLICKHOUSE_HOST=http://localhost:8123 CLICKHOUSE_USER=default CLICKHOUSE_PASSWORD= CLICKHOUSE_TZ= +CLICKHOUSE_MAX_EXECUTION_TIME=60 diff --git a/README.md b/README.md index bb0d6bb8..a3605456 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ To get the project up and running on your local machine, follow these steps: - `CLICKHOUSE_HOST`: ClickHouse host, for example `http://localhost:8123` - `CLICKHOUSE_USER`: ClickHouse user with permission to query the `system` database. - `CLICKHOUSE_PASSWORD`: ClickHouse password for the specified user. - - `CLICKHOUSE_TIMEOUT`: Timeout for ClickHouse queries in milliseconds. Default is `100000`. + - `CLICKHOUSE_MAX_EXECUTION_TIME`: [`max_execution_time`](https://clickhouse.com/docs/en/operations/settings/query-complexity#max-execution-time) timeout in seconds. Default is `10`. - `CLICKHOUSE_TZ`: ClickHouse server timezone. Default: `''`. 4. Run the development server with `npm run dev` or `yarn dev` 5. Open [http://localhost:3000](http://localhost:3000) in your browser to see the dashboard. @@ -50,6 +50,7 @@ docker run -it \ -e CLICKHOUSE_USER='default' \ -e CLICKHOUSE_PASSWORD='' \ -e CLICKHOUSE_TZ='Asia/Ho_Chi_Minh' \ + -e CLICKHOUSE_MAX_EXECUTION_TIME='15' \ --name clickhouse-monitoring \ ghcr.io/duyet/clickhouse-monitoring:main ``` @@ -71,6 +72,8 @@ env: value: '' - name: CLICKHOUSE_TZ value: 'Asia/Ho_Chi_Minh' + - name: CLICKHOUSE_MAX_EXECUTION_TIME + value: '15' EOF helm install -f values.yaml clickhouse-monitoring-release duyet/clickhouse-monitoring diff --git a/app/[query]/table.tsx b/app/[query]/table.tsx index 5d2a46fa..0f45dc04 100644 --- a/app/[query]/table.tsx +++ b/app/[query]/table.tsx @@ -58,10 +58,15 @@ export default async function Table({ ...config.defaultParams, ...validQueryParamsObj, } - const data = await fetchData({ + const { data } = await fetchData({ query: sql, format: 'JSONEachRow', query_params: queryParams, + clickhouse_settings: { + // The main data table takes longer to load. + max_execution_time: 300, + ...config.clickhouseSettings, + }, }) return diff --git a/app/api/timezone/route.ts b/app/api/timezone/route.ts index 9807cb0f..755e58cf 100644 --- a/app/api/timezone/route.ts +++ b/app/api/timezone/route.ts @@ -6,11 +6,12 @@ export const revalidate = false export async function GET() { try { - const resp = await fetchData<{ tz: string }[]>({ + const { data } = await fetchData<{ tz: string }[]>({ query: 'SELECT timezone() as tz', }) + return NextResponse.json({ - tz: resp[0].tz, + tz: data[0].tz, }) } catch { return NextResponse.json({}, { status: 500 }) diff --git a/app/api/version/route.ts b/app/api/version/route.ts index f0420e7d..9baf70b4 100644 --- a/app/api/version/route.ts +++ b/app/api/version/route.ts @@ -5,7 +5,10 @@ import { fetchData } from '@/lib/clickhouse' export async function GET() { try { - const clickhouse = await fetchData({ query: 'SELECT version()' }) + const { data: clickhouse } = await fetchData({ + query: 'SELECT version() as version', + }) + return NextResponse.json({ ui: packageInfo.version, clickhouse, diff --git a/app/clusters/[cluster]/breadcrumb.tsx b/app/clusters/[cluster]/breadcrumb.tsx index cdb48be3..088eb682 100644 --- a/app/clusters/[cluster]/breadcrumb.tsx +++ b/app/clusters/[cluster]/breadcrumb.tsx @@ -24,13 +24,11 @@ interface Props { } export async function ClusterListBreadcrumb({ cluster }: Props) { - let clusters: Row[] = [] - try { // Lists cluster names. - clusters = await fetchDataWithCache()({ query: config.sql }) + const { data } = await fetchDataWithCache()({ query: config.sql }) - if (!clusters.length) { + if (!data.length) { return ( ) } + + return } catch (e: any) { return ( ) } - - return } export function ClusterListBreadcrumbSkeleton({ cluster }: Props) { diff --git a/app/clusters/[cluster]/count-across-replicas/page.tsx b/app/clusters/[cluster]/count-across-replicas/page.tsx index 5e5451fd..e9bdc20e 100644 --- a/app/clusters/[cluster]/count-across-replicas/page.tsx +++ b/app/clusters/[cluster]/count-across-replicas/page.tsx @@ -11,7 +11,7 @@ interface PageProps { } export default async function Page({ params: { cluster } }: PageProps) { - const replicas = await fetchData<{ replica: string }[]>({ + const { data: replicas } = await fetchData<{ replica: string }[]>({ query: `SELECT hostName() as replica FROM clusterAllReplicas({cluster: String}) ORDER BY 1`, query_params: { cluster }, }) @@ -47,7 +47,7 @@ export default async function Page({ params: { cluster } }: PageProps) { }, } - const rows = await fetchData< + const { data } = await fetchData< { table: string [replica: string]: string | number @@ -61,7 +61,7 @@ export default async function Page({ params: { cluster } }: PageProps) { ) } diff --git a/app/clusters/[cluster]/parts-across-replicas/page.tsx b/app/clusters/[cluster]/parts-across-replicas/page.tsx index 2d0337af..b6287eba 100644 --- a/app/clusters/[cluster]/parts-across-replicas/page.tsx +++ b/app/clusters/[cluster]/parts-across-replicas/page.tsx @@ -11,7 +11,7 @@ interface PageProps { } export default async function Page({ params: { cluster } }: PageProps) { - const replicas = await fetchData<{ replica: string }[]>({ + const { data: replicas } = await fetchData<{ replica: string }[]>({ query: `SELECT hostName() as replica FROM clusterAllReplicas({cluster: String}) ORDER BY 1`, query_params: { cluster }, }) @@ -47,7 +47,7 @@ export default async function Page({ params: { cluster } }: PageProps) { }, } - const rows = await fetchData< + const { data } = await fetchData< { table: string [replica: string]: string | number @@ -61,7 +61,7 @@ export default async function Page({ params: { cluster } }: PageProps) { ) } diff --git a/app/clusters/[cluster]/replicas-status/page.tsx b/app/clusters/[cluster]/replicas-status/page.tsx index 1c51efe0..eed689cd 100644 --- a/app/clusters/[cluster]/replicas-status/page.tsx +++ b/app/clusters/[cluster]/replicas-status/page.tsx @@ -12,7 +12,7 @@ interface PageProps { } export default async function Page({ params: { cluster } }: PageProps) { - const tables = await fetchData({ + const { data } = await fetchData({ query: config.sql, query_params: { cluster }, }) @@ -21,7 +21,7 @@ export default async function Page({ params: { cluster } }: PageProps) { } /> ) diff --git a/app/clusters/page.tsx b/app/clusters/page.tsx index 49d3a66b..39c26d57 100644 --- a/app/clusters/page.tsx +++ b/app/clusters/page.tsx @@ -5,7 +5,7 @@ import { config, type Row } from './config' // Redirects to the first database. export default async function ClustersPage() { - const tables = await fetchData({ query: config.sql }) + const { data } = await fetchData({ query: config.sql }) - return + return } diff --git a/app/dashboard/render-chart.tsx b/app/dashboard/render-chart.tsx index b63d5278..c841c071 100644 --- a/app/dashboard/render-chart.tsx +++ b/app/dashboard/render-chart.tsx @@ -37,7 +37,7 @@ export const RenderChart = async ({ className, chartClassName, }: RenderChartProps) => { - const data = await fetchData<{ [key: string]: string | number }[]>({ + const { data } = await fetchData<{ [key: string]: string | number }[]>({ query, query_params: params, }) diff --git a/app/dashboard/seeding.ts b/app/dashboard/seeding.ts index 61e4fb51..c92ef3be 100644 --- a/app/dashboard/seeding.ts +++ b/app/dashboard/seeding.ts @@ -35,7 +35,7 @@ const migrateSettings = async () => { ] for (const seed of seeds) { - const exists = await fetchData({ + const { data: exists } = await fetchData({ query: ` SELECT * FROM ${TABLE_SETTINGS} FINAL @@ -99,7 +99,7 @@ const migrateDashboard = async () => { ] for (const seed of seeds) { - const exists = await fetchData({ + const { data: exists } = await fetchData({ query: `SELECT * FROM ${TABLE_CHARTS} FINAL WHERE title = '${seed.title}'`, }) diff --git a/app/dashboard/utils.ts b/app/dashboard/utils.ts index 0844b7e4..ec5bb437 100644 --- a/app/dashboard/utils.ts +++ b/app/dashboard/utils.ts @@ -10,14 +10,16 @@ import { seeding } from './seeding' export const getCustomDashboards = async () => { await seeding() - const dashboards = await fetchData({ + const q1 = fetchData({ query: `SELECT * FROM ${TABLE_CHARTS} FINAL ORDER BY ordering ASC`, - }) - const settings = await fetchData({ + }).then((res) => res.data) + const q2 = fetchData({ query: `SELECT * FROM ${TABLE_SETTINGS} FINAL`, - }) + }).then((res) => res.data) + + const [dashboards, settings] = await Promise.all([q1, q2]) - return { settings, dashboards } + return { dashboards, settings } } export async function updateSettingParams( diff --git a/app/database/[database]/[table]/@mergetree/page.tsx b/app/database/[database]/[table]/@mergetree/page.tsx index 09e1ea0e..11904682 100644 --- a/app/database/[database]/[table]/@mergetree/page.tsx +++ b/app/database/[database]/[table]/@mergetree/page.tsx @@ -22,7 +22,7 @@ export default async function MergeTree({ const engine = await engineType(database, table) if (engine.includes('MergeTree') === false) return <> - const columns = await fetchDataWithCache()({ + const { data: columns } = await fetchDataWithCache()({ query: config.sql, query_params: { database, @@ -30,9 +30,29 @@ export default async function MergeTree({ }, }) - let description = '' + return ( + } + toolbarExtras={} + topRightToolbarExtras={ + + } + config={config} + data={columns} + /> + ) +} + +async function Description({ + database, + table, +}: { + database: string + table: string +}) { try { - const raw = await fetchDataWithCache()<{ comment: string }[]>({ + const { data } = await fetchDataWithCache()<{ comment: string }[]>({ query: ` SELECT comment FROM system.tables @@ -42,23 +62,11 @@ export default async function MergeTree({ query_params: { database, table }, }) - description = raw?.[0]?.comment || '' + return data?.[0]?.comment || '' } catch (e) { console.error('Error fetching table description', e) + return '' } - - return ( - } - topRightToolbarExtras={ - - } - config={config} - data={columns} - /> - ) } const TopRightToolbarExtras = ({ diff --git a/app/database/[database]/[table]/engine-type.ts b/app/database/[database]/[table]/engine-type.ts index e6fa9782..f62ba043 100644 --- a/app/database/[database]/[table]/engine-type.ts +++ b/app/database/[database]/[table]/engine-type.ts @@ -2,7 +2,7 @@ import { fetchDataWithCache } from '@/lib/clickhouse' export const engineType = async (database: string, table: string) => { try { - const resp = await fetchDataWithCache()<{ engine: string }[]>({ + const { data } = await fetchDataWithCache()<{ engine: string }[]>({ query: ` SELECT engine FROM system.tables @@ -12,9 +12,9 @@ export const engineType = async (database: string, table: string) => { query_params: { database, table }, }) - return resp?.[0]?.engine || '' + return data?.[0]?.engine || '' } catch (error) { - console.error(error) + console.error(`Fetch engine type for ${database}.${table} error:`, error) return '' } } diff --git a/app/database/[database]/[table]/extras/alternative-tables.tsx b/app/database/[database]/[table]/extras/alternative-tables.tsx index 2b753220..838fcaee 100644 --- a/app/database/[database]/[table]/extras/alternative-tables.tsx +++ b/app/database/[database]/[table]/extras/alternative-tables.tsx @@ -22,10 +22,16 @@ export async function AlternativeTables({ }: AlternativeTablesProps) { let anotherTables: { name: string }[] = [] try { - anotherTables = await fetchData<{ name: string }[]>({ - query: `SELECT name FROM system.tables WHERE database = {database: String}`, + const res = await fetchData<{ name: string }[]>({ + query: ` + SELECT name + FROM system.tables + WHERE database = {database: String} + `, query_params: { database }, }) + + anotherTables = res.data } catch (error) { console.log(error) diff --git a/app/database/[database]/[table]/extras/running-queries-count.tsx b/app/database/[database]/[table]/extras/running-queries-count.tsx index c5e84f4e..950ac4a1 100644 --- a/app/database/[database]/[table]/extras/running-queries-count.tsx +++ b/app/database/[database]/[table]/extras/running-queries-count.tsx @@ -11,9 +11,8 @@ export async function RunningQueriesCount({ database, table, }: RunningQueriesProps) { - let data: { count: number }[] = [] try { - data = await fetchData<{ count: number }[]>({ + const { data } = await fetchData<{ count: number }[]>({ query: ` SELECT COUNT() as count FROM system.processes @@ -24,14 +23,14 @@ export async function RunningQueriesCount({ table, }, }) + + if (!data?.length) { + return null + } + + return {data[0].count || 0} } catch (error) { console.error(error) return null } - - if (!data?.length) { - return null - } - - return {data[0].count || 0} } diff --git a/app/database/[database]/[table]/extras/running-queries.tsx b/app/database/[database]/[table]/extras/running-queries.tsx index ea97e598..9f0643b3 100644 --- a/app/database/[database]/[table]/extras/running-queries.tsx +++ b/app/database/[database]/[table]/extras/running-queries.tsx @@ -21,7 +21,7 @@ export async function RunningQueries({ table, className, }: RunningQueriesProps) { - let data: { [key: string]: string }[] = [] + let data: { data: { [key: string]: string }[] } = { data: [] } try { data = await fetchData({ query: `SELECT query, user, elapsed, @@ -50,11 +50,11 @@ export async function RunningQueries({ ) } - if (!data?.length) { + if (!data.data?.length) { return No rows } - const headers = Object.keys(data[0]) + const headers = Object.keys(data.data[0]) return (
@@ -67,7 +67,7 @@ export async function RunningQueries({ - {data.map((row, idx) => ( + {data.data.map((row, idx) => ( {Object.values(row).map((value) => { if (typeof value === 'object') { diff --git a/app/database/[database]/[table]/extras/sample-data.tsx b/app/database/[database]/[table]/extras/sample-data.tsx index 1d6aa3ee..5c9233d5 100644 --- a/app/database/[database]/[table]/extras/sample-data.tsx +++ b/app/database/[database]/[table]/extras/sample-data.tsx @@ -22,7 +22,7 @@ export async function SampleData({ limit = 10, className, }: SampleDataProps) { - let data: { [key: string]: string }[] = [] + let data: { data: { [key: string]: string }[] } = { data: [] } try { data = await fetchDataWithCache()({ query: `SELECT * @@ -45,11 +45,11 @@ export async function SampleData({ ) } - if (!data?.length) { + if (!data.data?.length) { return No rows } - const headers = Object.keys(data[0]) + const headers = Object.keys(data.data[0]) return (
@@ -62,7 +62,7 @@ export async function SampleData({ - {data.map((row, idx) => ( + {data.data.map((row, idx) => ( {Object.values(row).map((value) => { if (typeof value === 'object') { diff --git a/app/database/[database]/[table]/extras/table-ddl.tsx b/app/database/[database]/[table]/extras/table-ddl.tsx index 4bf817d2..cd93d2ba 100644 --- a/app/database/[database]/[table]/extras/table-ddl.tsx +++ b/app/database/[database]/[table]/extras/table-ddl.tsx @@ -12,11 +12,12 @@ export async function TableDDL({ table, className, }: ShowSQLButtonProps) { - let showCreateTable: { statement: string }[] = [] + let showCreateTable = [] try { - showCreateTable = await fetchData({ + const { data }: { data: { statement: string }[] } = await fetchData({ query: `SHOW CREATE TABLE ${database}.${table}`, }) + showCreateTable = data } catch (error) { console.log(error) diff --git a/app/database/[database]/[table]/extras/table-info.tsx b/app/database/[database]/[table]/extras/table-info.tsx index 69a492c4..4e78ffa6 100644 --- a/app/database/[database]/[table]/extras/table-info.tsx +++ b/app/database/[database]/[table]/extras/table-info.tsx @@ -23,9 +23,9 @@ export async function TableInfo({ table, className, }: TableInfoProps) { - let tableInfo: { [key: string]: string }[] = [] + let tableInfo = [] try { - tableInfo = await fetchData({ + const { data }: { data: { [key: string]: string }[] } = await fetchData({ query: `SELECT engine, uuid, @@ -50,6 +50,7 @@ export async function TableInfo({ table, }, }) + tableInfo = data } catch (error) { console.log(error) diff --git a/app/database/[database]/breadcrumb.tsx b/app/database/[database]/breadcrumb.tsx index 395baaff..b04a0976 100644 --- a/app/database/[database]/breadcrumb.tsx +++ b/app/database/[database]/breadcrumb.tsx @@ -36,15 +36,18 @@ export async function DatabaseBreadcrumb({ database }: Props) { try { // List database names and number of tables - databases = await fetchDataWithCache()({ - query: listDatabases, - }) + const { data: databases }: { data: DatabaseCount[] } = + await fetchDataWithCache()({ + query: listDatabases, + }) if (!databases.length) { return ( ) } + + return } catch (e: any) { return ( ) } - - return } export async function DatabaseBreadcrumbSkeleton({ database }: Props) { diff --git a/app/database/[database]/page.tsx b/app/database/[database]/page.tsx index 38f2bf2b..5a38aa8b 100644 --- a/app/database/[database]/page.tsx +++ b/app/database/[database]/page.tsx @@ -50,7 +50,7 @@ export default async function TableListPage({ }, } - const tables = await fetchData({ + const { data } = await fetchData({ query: config.sql, format: 'JSONEachRow', query_params: { database }, @@ -60,7 +60,7 @@ export default async function TableListPage({ } /> ) diff --git a/app/explain/actions.ts b/app/explain/actions.ts index 2615fa5e..1fb9aeb5 100644 --- a/app/explain/actions.ts +++ b/app/explain/actions.ts @@ -52,7 +52,7 @@ export async function explainAction( const sql = `EXPLAIN ${kind} ${query}` try { - const data = await fetchData<{ explain: string }[]>({ + const { data } = await fetchData<{ explain: string }[]>({ query: sql, clickhouse_settings, }) diff --git a/app/replica/[replica]/tables/page.tsx b/app/replica/[replica]/tables/page.tsx index 6b4b572b..9a9b2638 100644 --- a/app/replica/[replica]/tables/page.tsx +++ b/app/replica/[replica]/tables/page.tsx @@ -10,7 +10,7 @@ interface PageProps { } export default async function ClustersPage({ params: { replica } }: PageProps) { - const tables = await fetchData({ + const { data } = await fetchData({ query: config.sql, query_params: { replica }, }) @@ -19,7 +19,7 @@ export default async function ClustersPage({ params: { replica } }: PageProps) { ) } diff --git a/components/charts/backup-size.tsx b/components/charts/backup-size.tsx index 5588da43..c1d7d0a3 100644 --- a/components/charts/backup-size.tsx +++ b/components/charts/backup-size.tsx @@ -24,7 +24,7 @@ export async function ChartBackupSize({ WHERE status = 'BACKUP_CREATED' ${startTimeCondition} ` - const data = await fetchData< + const { data } = await fetchData< { total_size: number uncompressed_size: number diff --git a/components/charts/connections-http.tsx b/components/charts/connections-http.tsx index 6894975a..647b2db5 100644 --- a/components/charts/connections-http.tsx +++ b/components/charts/connections-http.tsx @@ -27,7 +27,7 @@ export async function ChartConnectionsHttp({ ORDER BY event_time ` - const data = await fetchData< + const { data } = await fetchData< { event_time: string CurrentMetric_HTTPConnection: number diff --git a/components/charts/connections-interserver.tsx b/components/charts/connections-interserver.tsx index a8114510..23d8d242 100644 --- a/components/charts/connections-interserver.tsx +++ b/components/charts/connections-interserver.tsx @@ -22,7 +22,7 @@ export async function ChartConnectionsInterserver({ ORDER BY event_time ` - const data = await fetchData< + const { data } = await fetchData< { event_time: string CurrentMetric_InterserverConnection: number diff --git a/components/charts/cpu-usage.tsx b/components/charts/cpu-usage.tsx index 7bc9b918..2261edf1 100644 --- a/components/charts/cpu-usage.tsx +++ b/components/charts/cpu-usage.tsx @@ -17,7 +17,7 @@ export async function ChartCPUUsage({ WHERE event_time >= (now() - INTERVAL ${lastHours} HOUR) GROUP BY 1 ORDER BY 1` - const data = await fetchData<{ event_time: string; avg_cpu: number }[]>({ + const { data } = await fetchData<{ event_time: string; avg_cpu: number }[]>({ query, }) diff --git a/components/charts/disk-size.tsx b/components/charts/disk-size.tsx index d7e048d5..5eb17aaa 100644 --- a/components/charts/disk-size.tsx +++ b/components/charts/disk-size.tsx @@ -19,7 +19,7 @@ export async function ChartDiskSize({ ${condition} ORDER BY name ` - const data = await fetchData< + const { data } = await fetchData< { name: string used_space: number diff --git a/components/charts/disks-usage.tsx b/components/charts/disks-usage.tsx index 6f5b9b2d..9fc200af 100644 --- a/components/charts/disks-usage.tsx +++ b/components/charts/disks-usage.tsx @@ -26,7 +26,7 @@ export async function ChartDisksUsage({ ORDER BY 1 ASC ` - const data = await fetchData< + const { data } = await fetchData< { event_time: string DiskAvailable_default: number diff --git a/components/charts/memory-usage.tsx b/components/charts/memory-usage.tsx index 97ca6db5..c1420785 100644 --- a/components/charts/memory-usage.tsx +++ b/components/charts/memory-usage.tsx @@ -18,7 +18,7 @@ export async function ChartMemoryUsage({ WHERE event_time >= (now() - INTERVAL ${lastHours} HOUR) GROUP BY 1 ORDER BY 1 ASC` - const data = await fetchData< + const { data } = await fetchData< { event_time: string avg_memory: number diff --git a/components/charts/merge-avg-duration.tsx b/components/charts/merge-avg-duration.tsx index d38daa57..0ab9e442 100644 --- a/components/charts/merge-avg-duration.tsx +++ b/components/charts/merge-avg-duration.tsx @@ -23,7 +23,7 @@ export async function ChartMergeAvgDuration({ GROUP BY 1 ORDER BY 1 ASC ` - const data = await fetchData< + const { data } = await fetchData< { event_time: string avg_duration_ms: number diff --git a/components/charts/merge-count.tsx b/components/charts/merge-count.tsx index e92e946a..1230bdc4 100644 --- a/components/charts/merge-count.tsx +++ b/components/charts/merge-count.tsx @@ -24,7 +24,7 @@ export async function ChartMergeCount({ ORDER BY 1 ` - const data = await fetchData< + const { data } = await fetchData< { event_time: string avg_CurrentMetric_Merge: number diff --git a/components/charts/merge-sum-read-rows.tsx b/components/charts/merge-sum-read-rows.tsx index 76ff1f2c..bb51e367 100644 --- a/components/charts/merge-sum-read-rows.tsx +++ b/components/charts/merge-sum-read-rows.tsx @@ -23,7 +23,7 @@ export async function ChartMergeSumReadRows({ GROUP BY 1 ORDER BY 1 ASC ` - const data = await fetchData< + const { data } = await fetchData< { event_time: string sum_read_rows: number diff --git a/components/charts/new-parts-created.tsx b/components/charts/new-parts-created.tsx index 928a8fb1..7e6b25c8 100644 --- a/components/charts/new-parts-created.tsx +++ b/components/charts/new-parts-created.tsx @@ -30,7 +30,7 @@ export async function ChartNewPartsCreated({ table DESC ` - const raw = await fetchData< + const { data: raw } = await fetchData< { event_time: string table: string diff --git a/components/charts/query-count-by-user.tsx b/components/charts/query-count-by-user.tsx index fe7ef8b6..b93cb380 100644 --- a/components/charts/query-count-by-user.tsx +++ b/components/charts/query-count-by-user.tsx @@ -21,7 +21,7 @@ export async function ChartQueryCountByUser({ GROUP BY 1, 2 ORDER BY 1 ASC, 3 DESC ` - const raw = await fetchData< + const { data: raw } = await fetchData< { event_time: string user: string diff --git a/components/charts/query-count.tsx b/components/charts/query-count.tsx index 83a84128..9ebcf6af 100644 --- a/components/charts/query-count.tsx +++ b/components/charts/query-count.tsx @@ -45,7 +45,7 @@ export async function ChartQueryCount({ LEFT JOIN breakdown USING event_time ORDER BY 1 ` - const data = await fetchData< + const { data } = await fetchData< { event_time: string query_count: number diff --git a/components/charts/query-duration.tsx b/components/charts/query-duration.tsx index cc44e63a..da1a27a5 100644 --- a/components/charts/query-duration.tsx +++ b/components/charts/query-duration.tsx @@ -24,7 +24,7 @@ export async function ChartQueryDuration({ GROUP BY event_time ORDER BY event_time ASC ` - const data = await fetchData< + const { data } = await fetchData< { event_time: string query_duration_ms: number diff --git a/components/charts/query-memory.tsx b/components/charts/query-memory.tsx index c3fb8bae..946a9e11 100644 --- a/components/charts/query-memory.tsx +++ b/components/charts/query-memory.tsx @@ -24,7 +24,7 @@ export async function ChartQueryMemory({ GROUP BY event_time ORDER BY event_time ASC ` - const data = await fetchData< + const { data } = await fetchData< { event_time: string memory_usage: number diff --git a/components/charts/query-type.tsx b/components/charts/query-type.tsx index 32e9bf4a..c4379cc9 100644 --- a/components/charts/query-type.tsx +++ b/components/charts/query-type.tsx @@ -18,7 +18,7 @@ export async function ChartQueryType({ GROUP BY 1 ORDER BY 1 ` - const data = await fetchData< + const { data } = await fetchData< { type: string query_count: number diff --git a/components/charts/readonly-replica.tsx b/components/charts/readonly-replica.tsx index 711a9e57..3fc5bd91 100644 --- a/components/charts/readonly-replica.tsx +++ b/components/charts/readonly-replica.tsx @@ -19,7 +19,7 @@ export async function ChartReadonlyReplica({ ORDER BY event_time ` - const data = await fetchData< + const { data } = await fetchData< { event_time: string ReadonlyReplica: number diff --git a/components/charts/replication-queue-count.tsx b/components/charts/replication-queue-count.tsx index 93935132..5ef35d04 100644 --- a/components/charts/replication-queue-count.tsx +++ b/components/charts/replication-queue-count.tsx @@ -14,14 +14,14 @@ export async function ChartReplicationQueueCount({ countIf(is_currently_executing) AS count_executing FROM system.replication_queue ` - const rows = await fetchData< + const { data } = await fetchData< { count_all: number count_executing: number }[] >({ query }) - const count = rows?.[0] || { count_all: 0, count_executing: 0 } + const count = data?.[0] || { count_all: 0, count_executing: 0 } return ( ({ query: rowsReadWrittenSql }) - if (!!rows) { - rowsReadWritten = rows?.[0] + + if (!!data) { + rowsReadWritten = data?.[0] } } catch (e) { console.error('Error fetching rows read', e) @@ -99,7 +100,7 @@ export async function ChartSummaryUsedByMerges({ FROM system.merges ` try { - const rows = await fetchData< + const { data } = await fetchData< { bytes_read: number bytes_written: number @@ -108,8 +109,8 @@ export async function ChartSummaryUsedByMerges({ }[] >({ query: bytesReadWrittenSql }) - if (!!rows) { - bytesReadWritten = rows?.[0] + if (!!data) { + bytesReadWritten = data?.[0] } } catch (e) { console.error('Error fetching bytes read', e) diff --git a/components/charts/summary-used-by-mutations.tsx b/components/charts/summary-used-by-mutations.tsx index 1c85d61c..aef9f0e9 100644 --- a/components/charts/summary-used-by-mutations.tsx +++ b/components/charts/summary-used-by-mutations.tsx @@ -15,12 +15,12 @@ export async function ChartSummaryUsedByMutations({ FROM system.mutations WHERE is_done = 0 ` - const rows = await fetchData< + const { data } = await fetchData< { running_count: number }[] >({ query }) - const count = rows?.[0] || { running_count: 0 } + const count = data?.[0] || { running_count: 0 } const items: CardMultiMetricsProps['items'] = [] diff --git a/components/charts/summary-used-by-running-queries.tsx b/components/charts/summary-used-by-running-queries.tsx index 6f19a433..4d91d957 100644 --- a/components/charts/summary-used-by-running-queries.tsx +++ b/components/charts/summary-used-by-running-queries.tsx @@ -20,7 +20,7 @@ export async function ChartSummaryUsedByRunningQueries({ formatReadableSize(memory_usage) as readable_memory_usage FROM system.processes ` - const rows = await fetchData< + const { data } = await fetchData< { query_count: number memory_usage: number @@ -28,8 +28,8 @@ export async function ChartSummaryUsedByRunningQueries({ }[] >({ query: sql }) - const first = rows?.[0] - if (!rows || !first) return null + const first = data?.[0] + if (!data || !first) return null // Workaround for getting total memory usage const totalMemSql = ` @@ -47,7 +47,7 @@ export async function ChartSummaryUsedByRunningQueries({ readable_total: first.readable_memory_usage, } try { - const totalRows = await fetchData< + const { data: totalRows } = await fetchData< { metric: string total: number @@ -68,7 +68,7 @@ export async function ChartSummaryUsedByRunningQueries({ AND query_start_time >= today() ` try { - const todayQueryCountRows = await fetchData< + const { data: todayQueryCountRows } = await fetchData< { query_count: number }[] @@ -93,7 +93,7 @@ export async function ChartSummaryUsedByRunningQueries({ FROM system.processes ` try { - const rows = await fetchData< + const { data } = await fetchData< { rows_read: number rows_written: number @@ -101,8 +101,8 @@ export async function ChartSummaryUsedByRunningQueries({ readable_rows_written: string }[] >({ query: rowsReadWrittenSql }) - if (!!rows) { - rowsReadWritten = rows?.[0] + if (!!data) { + rowsReadWritten = data?.[0] } } catch (e) { console.error('Error fetching rows read', e) diff --git a/components/charts/top-table-size.tsx b/components/charts/top-table-size.tsx index 72f5c435..dd90fab0 100644 --- a/components/charts/top-table-size.tsx +++ b/components/charts/top-table-size.tsx @@ -39,7 +39,7 @@ export async function ChartTopTableSize({ GROUP BY 1 ORDER BY compressed_bytes DESC LIMIT ${limit}`, - }) + }).then((res) => res.data) const topByRowCountQuery = fetchData< { @@ -70,7 +70,7 @@ export async function ChartTopTableSize({ GROUP BY 1 ORDER BY total_rows DESC LIMIT ${limit}`, - }) + }).then((res) => res.data) const [topBySize, topByRowCount] = await Promise.all([ topBySizeQuery, diff --git a/components/charts/zookeeper-requests.tsx b/components/charts/zookeeper-requests.tsx index 576b428e..c2f1c5c6 100644 --- a/components/charts/zookeeper-requests.tsx +++ b/components/charts/zookeeper-requests.tsx @@ -22,7 +22,7 @@ export async function ChartZookeeperRequests({ ORDER BY event_time ` - const data = await fetchData< + const { data } = await fetchData< { event_time: string ZookeeperRequests: number diff --git a/components/charts/zookeeper-summary-table.tsx b/components/charts/zookeeper-summary-table.tsx index 2bd92b33..906039f0 100644 --- a/components/charts/zookeeper-summary-table.tsx +++ b/components/charts/zookeeper-summary-table.tsx @@ -20,7 +20,7 @@ export async function ChartZookeeperSummaryTable({ FROM system.metrics WHERE metric LIKE 'ZooKeeper%' ` - const data = await fetchData< + const { data } = await fetchData< { metric: string value: string diff --git a/components/charts/zookeeper-uptime.tsx b/components/charts/zookeeper-uptime.tsx index f72d207d..b550eabe 100644 --- a/components/charts/zookeeper-uptime.tsx +++ b/components/charts/zookeeper-uptime.tsx @@ -12,13 +12,13 @@ export async function ChartZookeeperUptime({ const query = 'SELECT formatReadableTimeDelta(zookeeperSessionUptime()) AS uptime' - const rows = await fetchData< + const { data } = await fetchData< { uptime: string }[] >({ query }) - const uptime = rows[0] || { uptime: 'N/A' } + const uptime = data[0] || { uptime: 'N/A' } return ( { let isOnline let title = [] try { - const detail = await fetchData< + const { data: detail } = await fetchData< { uptime: string; hostName: string; version: string }[] >({ query: ` SELECT formatReadableTimeDelta(uptime()) as uptime, hostName() as hostName, - version() as version`, + version() as version + `, }) isOnline = true diff --git a/components/data-table/cells/actions/actions.ts b/components/data-table/cells/actions/actions.ts index ab97e69b..baaf9194 100644 --- a/components/data-table/cells/actions/actions.ts +++ b/components/data-table/cells/actions/actions.ts @@ -50,14 +50,14 @@ export async function querySettings( _formData: FormData ): Promise { console.log('Getting query SETTINGS', queryId) - const res = await fetchData<{ Settings: string }[]>({ + const { data } = await fetchData<{ Settings: string }[]>({ query: `SELECT Settings FROM system.processes WHERE query_id = {queryId: String}`, query_params: { queryId }, }) - console.log('Query SETTINGS', queryId, res) + console.log('Query SETTINGS', queryId, data) return { action: 'toast', - message: JSON.stringify(res), + message: JSON.stringify(data), } } diff --git a/components/data-table/data-table.tsx b/components/data-table/data-table.tsx index 1a53b1bd..c1ce4914 100644 --- a/components/data-table/data-table.tsx +++ b/components/data-table/data-table.tsx @@ -36,7 +36,7 @@ import { DataTableToolbar } from './data-table-toolbar' interface DataTableProps { title?: string - description?: string + description?: string | React.ReactNode toolbarExtras?: React.ReactNode topRightToolbarExtras?: React.ReactNode config: QueryConfig diff --git a/components/menu/count-badge.tsx b/components/menu/count-badge.tsx index 2614c92e..0158373d 100644 --- a/components/menu/count-badge.tsx +++ b/components/menu/count-badge.tsx @@ -14,29 +14,27 @@ export async function CountBadge({ }: CountBadgeProps): Promise { if (!sql) return null - let data: any[] = [] - try { - data = await fetchData<{ 'count()': string }[]>({ + const { data } = await fetchData<{ 'count()': string }[]>({ query: sql, format: 'JSONEachRow', clickhouse_settings: { use_query_cache: 1, query_cache_ttl: 120 }, }) + + if (!data || !data.length || !data?.[0]?.['count()']) return null + + const count = data[0]['count()'] || 0 + if (count == 0) return null + + return ( + + {count} + + ) } catch (e: any) { console.error( `: could not get count for sql: ${sql}, error: ${e}` ) return null } - - if (!data || !data.length || !data?.[0]?.['count()']) return null - - const count = data[0]['count()'] || data[0]['count'] || 0 - if (count == 0) return null - - return ( - - {count} - - ) } diff --git a/lib/clickhouse.ts b/lib/clickhouse.ts index c16a75ef..4101bc49 100644 --- a/lib/clickhouse.ts +++ b/lib/clickhouse.ts @@ -1,10 +1,12 @@ import type { ClickHouseClient, DataFormat } from '@clickhouse/client' import { createClient } from '@clickhouse/client' -import type { QueryParams } from '@clickhouse/client-common' +import type { ClickHouseSettings, QueryParams } from '@clickhouse/client-common' import { createClient as createClientWeb } from '@clickhouse/client-web' import type { WebClickHouseClient } from '@clickhouse/client-web/dist/client' import { cache } from 'react' +const DEFAULT_CLICKHOUSE_MAX_EXECUTION_TIME = '60' + export const getClickHouseHosts = () => { const hosts = (process.env.CLICKHOUSE_HOST || '') .split(',') @@ -18,8 +20,10 @@ export const getClickHouseHost = () => getClickHouseHosts()[0] export const getClient = ({ web, + clickhouse_settings, }: { web?: B + clickhouse_settings?: ClickHouseSettings }): B extends true ? WebClickHouseClient : ClickHouseClient => { const client = web === true ? createClientWeb : createClient @@ -27,7 +31,13 @@ export const getClient = ({ host: getClickHouseHost(), username: process.env.CLICKHOUSE_USER ?? 'default', password: process.env.CLICKHOUSE_PASSWORD ?? '', - request_timeout: parseInt(process.env.CLICKHOUSE_TIMEOUT ?? '100000'), + clickhouse_settings: { + max_execution_time: parseInt( + process.env.CLICKHOUSE_MAX_EXECUTION_TIME ?? + DEFAULT_CLICKHOUSE_MAX_EXECUTION_TIME + ), + ...clickhouse_settings, + }, }) as B extends true ? WebClickHouseClient : ClickHouseClient } @@ -44,7 +54,7 @@ export const fetchData = async < query_params, format = 'JSONEachRow', clickhouse_settings, -}: QueryParams): Promise => { +}: QueryParams): Promise<{ data: T }> => { const start = new Date() const client = getClient({ web: false }) @@ -94,7 +104,7 @@ export const fetchData = async < console.debug(`<-- Response (${query_id}):`, data, '\n') } - return data + return { data } } export const fetchDataWithCache = () => cache(fetchData) diff --git a/lib/types/query-config.ts b/lib/types/query-config.ts index 4da886ef..a09558aa 100644 --- a/lib/types/query-config.ts +++ b/lib/types/query-config.ts @@ -5,6 +5,7 @@ import type { } from '@/components/data-table/column-defs' import type { PartialBy } from '@/lib/types/generic' import type { Icon } from '@/lib/types/icon' +import type { ClickHouseSettings } from '@clickhouse/client' export interface QueryConfig { name: string @@ -63,6 +64,10 @@ export interface QueryConfig { sql: string icon?: Icon }[] + /** + * ClickHouse settings to be used for this query + */ + clickhouseSettings?: ClickHouseSettings } export type QueryConfigNoName = PartialBy