From ef4e6340fdb42e7bd0e9181cab6e136762092be1 Mon Sep 17 00:00:00 2001 From: Eugene Klimov Date: Wed, 25 Dec 2024 19:15:04 +0500 Subject: [PATCH 1/3] Add `adhoc hide table names` connection settings option, fix https://github.com/Altinity/clickhouse-grafana/issues/456 Signed-off-by: Eugene Klimov --- CHANGELOG.md | 3 +- .../adhoc_hide_table_names_issue_456.json | 187 ++++++++++++++++++ .../clickhouse-hide-adhoc-tables.yaml | 18 ++ src/datasource/adhoc.ts | 59 ++++-- src/datasource/datasource.ts | 2 + src/types/types.ts | 3 +- src/views/ConfigEditor/ConfigEditor.tsx | 10 +- 7 files changed, 268 insertions(+), 14 deletions(-) create mode 100644 docker/grafana/dashboards/adhoc_hide_table_names_issue_456.json create mode 100644 docker/grafana/provisioning/datasources/clickhouse-hide-adhoc-tables.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 06997da1d..3a5fa4f16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,12 @@ ## Enhancements: * Add using window functions instead of `runningDifference` and `neighbor` for macros, to avoid `allow_deprecated_error_prone_window_functions`, fix https://github.com/Altinity/clickhouse-grafana/issues/572 * Add public coverage report summary, fix https://github.com/Altinity/clickhouse-grafana/issues/660 -* Add support DateTime(timezone) types to Annotations query, fix https://github.com/Altinity/clickhouse-grafana/issues/642 +* Add support `DateTime(timezone)` types to Annotations query, fix https://github.com/Altinity/clickhouse-grafana/issues/642 * Add single stat panel with categories, fix https://github.com/Altinity/clickhouse-grafana/issues/403 * Add log context windows size to connection settings, fix https://github.com/Altinity/clickhouse-grafana/issues/657 * Add `X-ClickHouse-SSL-Certificate-Auth` support, fix https://github.com/Altinity/clickhouse-grafana/issues/580 * Add `$columnsMs` macro, fix https://github.com/Altinity/clickhouse-grafana/issues/430 +* Add `adhoc hide table names` connection settings option, fix https://github.com/Altinity/clickhouse-grafana/issues/456 ## Fixes: * Add transposed table example, fix https://github.com/Altinity/clickhouse-grafana/issues/404 diff --git a/docker/grafana/dashboards/adhoc_hide_table_names_issue_456.json b/docker/grafana/dashboards/adhoc_hide_table_names_issue_456.json new file mode 100644 index 000000000..acbff4d4f --- /dev/null +++ b/docker/grafana/dashboards/adhoc_hide_table_names_issue_456.json @@ -0,0 +1,187 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 43, + "links": [], + "panels": [ + { + "datasource": { + "type": "vertamedia-clickhouse-datasource", + "uid": "P94617B551CAC0F6A" + }, + "description": "reproduce https://github.com/Altinity/clickhouse-grafana/issues/456", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "adHocFilters": [ + { + "condition": "", + "key": "service_name", + "keyLabel": "service_name", + "operator": "=", + "value": "mysql", + "valueLabels": [ + "mysql" + ] + } + ], + "adHocValuesQuery": "", + "add_metadata": true, + "contextWindowSize": "10", + "database": "default", + "datasource": { + "type": "vertamedia-clickhouse-datasource", + "uid": "P94617B551CAC0F6A" + }, + "dateTimeColDataType": "event_time", + "dateTimeType": "DATETIME", + "editorMode": "sql", + "extrapolate": true, + "format": "time_series", + "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", + "interval": "", + "intervalFactor": 1, + "query": "SELECT $timeSeries as t, service_name, count() FROM $table WHERE $timeFilter GROUP BY t, service_name ORDER BY t", + "rawQuery": " /* grafana dashboard=AdHoc with Hiding table name, user=0 */\n\nSELECT\n (intDiv(toUInt32(event_time), 20) * 20) * 1000 as t,\n service_name,\n count()\nFROM default.test_grafana\n\nWHERE\n event_time >= toDateTime(1735113709) AND event_time <= toDateTime(1735135309)\n AND service_name = 'service990'\nGROUP BY\n t,\n service_name\nORDER BY t\n", + "refId": "A", + "round": "0s", + "showFormattedSQL": true, + "skip_comments": true, + "table": "test_grafana", + "useWindowFuncForMacros": true + } + ], + "title": "AdHoc with Hide Table name", + "type": "timeseries" + } + ], + "preload": false, + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "baseFilters": [], + "datasource": { + "type": "vertamedia-clickhouse-datasource", + "uid": "P94617B551CAC0F6A" + }, + "filters": [ + { + "condition": "", + "key": "service_name", + "keyLabel": "service_name", + "operator": "=", + "value": "mysql", + "valueLabels": [ + "mysql" + ] + } + ], + "name": "adhoc_hide_tables", + "type": "adhoc" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "AdHoc with Hiding table name", + "uid": "fe7zdj034j1tsc", + "version": 1, + "weekStart": "" +} diff --git a/docker/grafana/provisioning/datasources/clickhouse-hide-adhoc-tables.yaml b/docker/grafana/provisioning/datasources/clickhouse-hide-adhoc-tables.yaml new file mode 100644 index 000000000..9f3c41c08 --- /dev/null +++ b/docker/grafana/provisioning/datasources/clickhouse-hide-adhoc-tables.yaml @@ -0,0 +1,18 @@ +apiVersion: 1 + +datasources: + - name: clickhouse-hide-adhoc-tables + type: vertamedia-clickhouse-datasource + access: proxy + url: http://clickhouse:8123 + basicAuth: true + secureJsonData: + basicAuthPassword: "" + basicAuthUser: "default" + editable: false + jsonData: + addCorsHeader: true + usePOST: true + useCompression: true + compressionType: gzip + adHocHideTableNames: true diff --git a/src/datasource/adhoc.ts b/src/datasource/adhoc.ts index e5cc7938b..097ef04c5 100644 --- a/src/datasource/adhoc.ts +++ b/src/datasource/adhoc.ts @@ -17,7 +17,7 @@ export default class AdHocFilter { this.adHocValuesQuery = datasource.adHocValuesQuery; let filter = queryFilter; if (datasource.defaultDatabase.length > 0) { - filter = "database = '" + datasource.defaultDatabase + "' AND " + queryFilter; + filter = "database = '" + datasource.defaultDatabase + "'"; } this.query = columnsQuery.replace('{filter}', filter); } @@ -47,22 +47,28 @@ export default class AdHocFilter { const databasePrefix = this.datasource.defaultDatabase.length === 0 ? item.database + '.' : ''; const text: string = databasePrefix + item.table + '.' + item.name; - this.tagKeys.push({ text: text, value: text }); - + if (!this.datasource.adHocHideTableNames) { + this.tagKeys.push({ text: text, value: text }); + } if (item.type.slice(0, 4) === 'Enum') { const regexEnum = /'(?:[^']+|'')+'/gim; - const options = item.type.match(regexEnum) || []; - - if (options.length > 0) { - this.tagValues[text] = options.map((o: any) => ({ text: o, value: o })); - this.tagValues[item.name] = this.tagValues[text]; + const enumValues = item.type.match(regexEnum) || []; + + if (enumValues.length > 0) { + if (!this.datasource.adHocHideTableNames) { + this.tagValues[text] = enumValues.map((o: any) => ({ text: o, value: o })); + } + if (!this.tagValues[item.name]) { + this.tagValues[item.name] = this.tagValues[text]; + } else { + this.tagValues[item.name].combine(this.tagValues[text]); + } } } - columnNames[item.name] = true; }); - // Store unique column names with wildcard table + // Store unique column names without table name Object.keys(columnNames).forEach((columnName) => { this.tagKeys.push({ text: columnName, value: columnName }); }); @@ -73,7 +79,38 @@ export default class AdHocFilter { // GetTagValues returns column values according to passed options // Values for fields with Enum type were already fetched in GetTagKeys func and stored in `tagValues` // Values for fields which not represented on `tagValues` get from ClickHouse and cached on `tagValues` - GetTagValues(options) { + async GetTagValues(options) { + if (this.datasource.adHocHideTableNames) { + // @todo could be very slow + const allTablesColumnSQL = "SELECT groupConcat(' UNION ALL ')(concat('( SELECT DISTINCT toString(`',name,'`) AS value FROM `',database,'`.`',table,'` LIMIT 300 )')) AS q FROM system.columns WHERE name='"+options.key+"'" + let allValuesSQL = ''; + let isGetAllValuesOK: boolean = await this.datasource + .metricFindQuery(allTablesColumnSQL) + .then((response: any) => { + allValuesSQL = response[0].text; + return true; + }) + .catch((error: any) => { + console.error(error); + return false; + }); + + if (!isGetAllValuesOK) { + return [] + } + return this.datasource + .metricFindQuery(allValuesSQL) + .then((response: any) => { + // Process and cache the response + this.tagValues[options.key] = this.processTagValuesResponse(response); + return this.tagValues[options.key]; + }) + .catch((error: any) => { + this.tagValues[options.key] = []; + console.error(error); + return this.tagValues[options.key]; + }); + } // Determine which query to use initially const initialQuery = this.adHocValuesQuery || DEFAULT_VALUES_QUERY; diff --git a/src/datasource/datasource.ts b/src/datasource/datasource.ts index 5a60cdf90..68976661e 100644 --- a/src/datasource/datasource.ts +++ b/src/datasource/datasource.ts @@ -50,6 +50,7 @@ export class CHDataSource useCompression: boolean; compressionType: string; adHocValuesQuery: string; + adHocHideTableNames: boolean; uid: string; constructor(instanceSettings: DataSourceInstanceSettings) { @@ -62,6 +63,7 @@ export class CHDataSource this.usePOST = instanceSettings.jsonData.usePOST || false; this.useCompression = instanceSettings.jsonData.useCompression || false; this.adHocValuesQuery = instanceSettings.jsonData.adHocValuesQuery || ''; + this.adHocHideTableNames = instanceSettings.jsonData.adHocHideTableNames || false; this.compressionType = instanceSettings.jsonData.compressionType || ''; this.defaultDatabase = instanceSettings.jsonData.defaultDatabase || ''; this.xHeaderUser = instanceSettings.jsonData.xHeaderUser || ''; diff --git a/src/types/types.ts b/src/types/types.ts index 59856d132..231e758e6 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -75,7 +75,8 @@ export interface CHDataSourceOptions extends DataSourceJsonData { defaultTimeStamp64_3?: string; defaultTimeStamp64_6?: string; defaultTimeStamp64_9?: string; - adHocValuesQuery: string; + adHocValuesQuery?: string; + adHocHideTableNames?: boolean; contextWindowSize?: string; useWindowFuncForMacros?: boolean; } diff --git a/src/views/ConfigEditor/ConfigEditor.tsx b/src/views/ConfigEditor/ConfigEditor.tsx index 83e9813e9..738c53297 100644 --- a/src/views/ConfigEditor/ConfigEditor.tsx +++ b/src/views/ConfigEditor/ConfigEditor.tsx @@ -42,7 +42,7 @@ export function ConfigEditor(props: Props) { const onSwitchToggle = ( key: keyof Pick< CHDataSourceOptions, - 'useYandexCloudAuthorization' | 'addCorsHeader' | 'usePOST' | 'useCompression' | 'xClickHouseSSLCertificateAuth' + 'useYandexCloudAuthorization' | 'addCorsHeader' | 'usePOST' | 'useCompression' | 'xClickHouseSSLCertificateAuth' | 'adHocHideTableNames' >, value: boolean ) => { @@ -228,6 +228,14 @@ export function ConfigEditor(props: Props) { /> + + onSwitchToggle('adHocHideTableNames', e.currentTarget.checked)} + /> + ); From 1c63d01d389f088aaaf16ee8e23cfac8d0335579 Mon Sep 17 00:00:00 2001 From: Eugene Klimov Date: Wed, 25 Dec 2024 19:28:27 +0500 Subject: [PATCH 2/3] fix tests for https://github.com/Altinity/clickhouse-grafana/issues/456 Signed-off-by: Eugene Klimov --- src/spec/datasource.jest.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spec/datasource.jest.ts b/src/spec/datasource.jest.ts index 198117296..986e7bf42 100644 --- a/src/spec/datasource.jest.ts +++ b/src/spec/datasource.jest.ts @@ -172,7 +172,7 @@ describe('clickhouse sql series:', () => { let adhocCtrl = new AdhocCtrl({ defaultDatabase: 'default' }); it('should be inited', function () { expect(adhocCtrl.query).toBe( - "SELECT database, table, name, type FROM system.columns WHERE database = 'default' AND database NOT IN ('system','INFORMATION_SCHEMA','information_schema') ORDER BY database, table" + "SELECT database, table, name, type FROM system.columns WHERE database = 'default' ORDER BY database, table" ); expect(adhocCtrl.datasource.defaultDatabase).toBe('default'); }); From 39d3ced343141cb1d0510ff01cdfc9bdcb9ff0f8 Mon Sep 17 00:00:00 2001 From: Eugene Klimov Date: Thu, 26 Dec 2024 17:14:46 +0500 Subject: [PATCH 3/3] apply AdHoc SQL query when hide adhoc table names, advanced fix for https://github.com/Altinity/clickhouse-grafana/issues/456 Signed-off-by: Eugene Klimov --- .../adhoc_hide_table_names_issue_456.json | 15 ++-------- src/datasource/adhoc.ts | 28 +++++++++++-------- 2 files changed, 19 insertions(+), 24 deletions(-) diff --git a/docker/grafana/dashboards/adhoc_hide_table_names_issue_456.json b/docker/grafana/dashboards/adhoc_hide_table_names_issue_456.json index acbff4d4f..cf0476d84 100644 --- a/docker/grafana/dashboards/adhoc_hide_table_names_issue_456.json +++ b/docker/grafana/dashboards/adhoc_hide_table_names_issue_456.json @@ -104,18 +104,7 @@ "pluginVersion": "11.4.0", "targets": [ { - "adHocFilters": [ - { - "condition": "", - "key": "service_name", - "keyLabel": "service_name", - "operator": "=", - "value": "mysql", - "valueLabels": [ - "mysql" - ] - } - ], + "adHocFilters": [], "adHocValuesQuery": "", "add_metadata": true, "contextWindowSize": "10", @@ -181,7 +170,7 @@ "timepicker": {}, "timezone": "browser", "title": "AdHoc with Hiding table name", - "uid": "fe7zdj034j1tsc", + "uid": "adhoc_hide_table_names", "version": 1, "weekStart": "" } diff --git a/src/datasource/adhoc.ts b/src/datasource/adhoc.ts index 097ef04c5..6f62e1508 100644 --- a/src/datasource/adhoc.ts +++ b/src/datasource/adhoc.ts @@ -80,14 +80,26 @@ export default class AdHocFilter { // Values for fields with Enum type were already fetched in GetTagKeys func and stored in `tagValues` // Values for fields which not represented on `tagValues` get from ClickHouse and cached on `tagValues` async GetTagValues(options) { + // Determine which query to use initially + const initialQuery = this.adHocValuesQuery || DEFAULT_VALUES_QUERY; + // Function to build the query + let database: string, table: string, field: string; + const buildQuery = (queryTemplate: string) => + queryTemplate.replace('{field}', field).replace('{database}', database).replace('{table}', table); + if (this.datasource.adHocHideTableNames) { // @todo could be very slow - const allTablesColumnSQL = "SELECT groupConcat(' UNION ALL ')(concat('( SELECT DISTINCT toString(`',name,'`) AS value FROM `',database,'`.`',table,'` LIMIT 300 )')) AS q FROM system.columns WHERE name='"+options.key+"'" - let allValuesSQL = ''; + const allTablesColumnSQL = "SELECT name,database,table FROM system.columns WHERE name='" + options.key + "'"; + let allValuesSQL: string[] = []; let isGetAllValuesOK: boolean = await this.datasource .metricFindQuery(allTablesColumnSQL) .then((response: any) => { - allValuesSQL = response[0].text; + allValuesSQL = response.map((item: any) => { + field = item.name; + database = item.database; + table = item.table; + return buildQuery("(" + initialQuery + ")"); + }); return true; }) .catch((error: any) => { @@ -96,10 +108,10 @@ export default class AdHocFilter { }); if (!isGetAllValuesOK) { - return [] + return []; } return this.datasource - .metricFindQuery(allValuesSQL) + .metricFindQuery(allValuesSQL.join(" UNION ALL ")) .then((response: any) => { // Process and cache the response this.tagValues[options.key] = this.processTagValuesResponse(response); @@ -111,8 +123,6 @@ export default class AdHocFilter { return this.tagValues[options.key]; }); } - // Determine which query to use initially - const initialQuery = this.adHocValuesQuery || DEFAULT_VALUES_QUERY; // If the tag values are already cached, return them immediately if (Object.prototype.hasOwnProperty.call(this.tagValues, options.key)) { @@ -125,7 +135,6 @@ export default class AdHocFilter { } // Destructure key items based on their length - let database, table, field; if (keyItems.length === 3) { [database, table, field] = keyItems; } else { @@ -133,9 +142,6 @@ export default class AdHocFilter { [table, field] = keyItems; } - // Function to build the query - const buildQuery = (queryTemplate) => - queryTemplate.replace('{field}', field).replace('{database}', database).replace('{table}', table); // Execute the initial query return this.datasource