diff --git a/CHANGELOG.md b/CHANGELOG.md index 06997da1d..3a5fa4f16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,12 @@ ## Enhancements: * Add using window functions instead of `runningDifference` and `neighbor` for macros, to avoid `allow_deprecated_error_prone_window_functions`, fix https://github.com/Altinity/clickhouse-grafana/issues/572 * Add public coverage report summary, fix https://github.com/Altinity/clickhouse-grafana/issues/660 -* Add support DateTime(timezone) types to Annotations query, fix https://github.com/Altinity/clickhouse-grafana/issues/642 +* Add support `DateTime(timezone)` types to Annotations query, fix https://github.com/Altinity/clickhouse-grafana/issues/642 * Add single stat panel with categories, fix https://github.com/Altinity/clickhouse-grafana/issues/403 * Add log context windows size to connection settings, fix https://github.com/Altinity/clickhouse-grafana/issues/657 * Add `X-ClickHouse-SSL-Certificate-Auth` support, fix https://github.com/Altinity/clickhouse-grafana/issues/580 * Add `$columnsMs` macro, fix https://github.com/Altinity/clickhouse-grafana/issues/430 +* Add `adhoc hide table names` connection settings option, fix https://github.com/Altinity/clickhouse-grafana/issues/456 ## Fixes: * Add transposed table example, fix https://github.com/Altinity/clickhouse-grafana/issues/404 diff --git a/docker/grafana/dashboards/adhoc_hide_table_names_issue_456.json b/docker/grafana/dashboards/adhoc_hide_table_names_issue_456.json new file mode 100644 index 000000000..cf0476d84 --- /dev/null +++ b/docker/grafana/dashboards/adhoc_hide_table_names_issue_456.json @@ -0,0 +1,176 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 43, + "links": [], + "panels": [ + { + "datasource": { + "type": "vertamedia-clickhouse-datasource", + "uid": "P94617B551CAC0F6A" + }, + "description": "reproduce https://github.com/Altinity/clickhouse-grafana/issues/456", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "adHocFilters": [], + "adHocValuesQuery": "", + "add_metadata": true, + "contextWindowSize": "10", + "database": "default", + "datasource": { + "type": "vertamedia-clickhouse-datasource", + "uid": "P94617B551CAC0F6A" + }, + "dateTimeColDataType": "event_time", + "dateTimeType": "DATETIME", + "editorMode": "sql", + "extrapolate": true, + "format": "time_series", + "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", + "interval": "", + "intervalFactor": 1, + "query": "SELECT $timeSeries as t, service_name, count() FROM $table WHERE $timeFilter GROUP BY t, service_name ORDER BY t", + "rawQuery": " /* grafana dashboard=AdHoc with Hiding table name, user=0 */\n\nSELECT\n (intDiv(toUInt32(event_time), 20) * 20) * 1000 as t,\n service_name,\n count()\nFROM default.test_grafana\n\nWHERE\n event_time >= toDateTime(1735113709) AND event_time <= toDateTime(1735135309)\n AND service_name = 'service990'\nGROUP BY\n t,\n service_name\nORDER BY t\n", + "refId": "A", + "round": "0s", + "showFormattedSQL": true, + "skip_comments": true, + "table": "test_grafana", + "useWindowFuncForMacros": true + } + ], + "title": "AdHoc with Hide Table name", + "type": "timeseries" + } + ], + "preload": false, + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "baseFilters": [], + "datasource": { + "type": "vertamedia-clickhouse-datasource", + "uid": "P94617B551CAC0F6A" + }, + "filters": [ + { + "condition": "", + "key": "service_name", + "keyLabel": "service_name", + "operator": "=", + "value": "mysql", + "valueLabels": [ + "mysql" + ] + } + ], + "name": "adhoc_hide_tables", + "type": "adhoc" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "AdHoc with Hiding table name", + "uid": "adhoc_hide_table_names", + "version": 1, + "weekStart": "" +} diff --git a/docker/grafana/provisioning/datasources/clickhouse-hide-adhoc-tables.yaml b/docker/grafana/provisioning/datasources/clickhouse-hide-adhoc-tables.yaml new file mode 100644 index 000000000..9f3c41c08 --- /dev/null +++ b/docker/grafana/provisioning/datasources/clickhouse-hide-adhoc-tables.yaml @@ -0,0 +1,18 @@ +apiVersion: 1 + +datasources: + - name: clickhouse-hide-adhoc-tables + type: vertamedia-clickhouse-datasource + access: proxy + url: http://clickhouse:8123 + basicAuth: true + secureJsonData: + basicAuthPassword: "" + basicAuthUser: "default" + editable: false + jsonData: + addCorsHeader: true + usePOST: true + useCompression: true + compressionType: gzip + adHocHideTableNames: true diff --git a/src/datasource/adhoc.ts b/src/datasource/adhoc.ts index e5cc7938b..6f62e1508 100644 --- a/src/datasource/adhoc.ts +++ b/src/datasource/adhoc.ts @@ -17,7 +17,7 @@ export default class AdHocFilter { this.adHocValuesQuery = datasource.adHocValuesQuery; let filter = queryFilter; if (datasource.defaultDatabase.length > 0) { - filter = "database = '" + datasource.defaultDatabase + "' AND " + queryFilter; + filter = "database = '" + datasource.defaultDatabase + "'"; } this.query = columnsQuery.replace('{filter}', filter); } @@ -47,22 +47,28 @@ export default class AdHocFilter { const databasePrefix = this.datasource.defaultDatabase.length === 0 ? item.database + '.' : ''; const text: string = databasePrefix + item.table + '.' + item.name; - this.tagKeys.push({ text: text, value: text }); - + if (!this.datasource.adHocHideTableNames) { + this.tagKeys.push({ text: text, value: text }); + } if (item.type.slice(0, 4) === 'Enum') { const regexEnum = /'(?:[^']+|'')+'/gim; - const options = item.type.match(regexEnum) || []; - - if (options.length > 0) { - this.tagValues[text] = options.map((o: any) => ({ text: o, value: o })); - this.tagValues[item.name] = this.tagValues[text]; + const enumValues = item.type.match(regexEnum) || []; + + if (enumValues.length > 0) { + if (!this.datasource.adHocHideTableNames) { + this.tagValues[text] = enumValues.map((o: any) => ({ text: o, value: o })); + } + if (!this.tagValues[item.name]) { + this.tagValues[item.name] = this.tagValues[text]; + } else { + this.tagValues[item.name].combine(this.tagValues[text]); + } } } - columnNames[item.name] = true; }); - // Store unique column names with wildcard table + // Store unique column names without table name Object.keys(columnNames).forEach((columnName) => { this.tagKeys.push({ text: columnName, value: columnName }); }); @@ -73,9 +79,50 @@ export default class AdHocFilter { // GetTagValues returns column values according to passed options // Values for fields with Enum type were already fetched in GetTagKeys func and stored in `tagValues` // Values for fields which not represented on `tagValues` get from ClickHouse and cached on `tagValues` - GetTagValues(options) { + async GetTagValues(options) { // Determine which query to use initially const initialQuery = this.adHocValuesQuery || DEFAULT_VALUES_QUERY; + // Function to build the query + let database: string, table: string, field: string; + const buildQuery = (queryTemplate: string) => + queryTemplate.replace('{field}', field).replace('{database}', database).replace('{table}', table); + + if (this.datasource.adHocHideTableNames) { + // @todo could be very slow + const allTablesColumnSQL = "SELECT name,database,table FROM system.columns WHERE name='" + options.key + "'"; + let allValuesSQL: string[] = []; + let isGetAllValuesOK: boolean = await this.datasource + .metricFindQuery(allTablesColumnSQL) + .then((response: any) => { + allValuesSQL = response.map((item: any) => { + field = item.name; + database = item.database; + table = item.table; + return buildQuery("(" + initialQuery + ")"); + }); + return true; + }) + .catch((error: any) => { + console.error(error); + return false; + }); + + if (!isGetAllValuesOK) { + return []; + } + return this.datasource + .metricFindQuery(allValuesSQL.join(" UNION ALL ")) + .then((response: any) => { + // Process and cache the response + this.tagValues[options.key] = this.processTagValuesResponse(response); + return this.tagValues[options.key]; + }) + .catch((error: any) => { + this.tagValues[options.key] = []; + console.error(error); + return this.tagValues[options.key]; + }); + } // If the tag values are already cached, return them immediately if (Object.prototype.hasOwnProperty.call(this.tagValues, options.key)) { @@ -88,7 +135,6 @@ export default class AdHocFilter { } // Destructure key items based on their length - let database, table, field; if (keyItems.length === 3) { [database, table, field] = keyItems; } else { @@ -96,9 +142,6 @@ export default class AdHocFilter { [table, field] = keyItems; } - // Function to build the query - const buildQuery = (queryTemplate) => - queryTemplate.replace('{field}', field).replace('{database}', database).replace('{table}', table); // Execute the initial query return this.datasource diff --git a/src/datasource/datasource.ts b/src/datasource/datasource.ts index 5a60cdf90..68976661e 100644 --- a/src/datasource/datasource.ts +++ b/src/datasource/datasource.ts @@ -50,6 +50,7 @@ export class CHDataSource useCompression: boolean; compressionType: string; adHocValuesQuery: string; + adHocHideTableNames: boolean; uid: string; constructor(instanceSettings: DataSourceInstanceSettings) { @@ -62,6 +63,7 @@ export class CHDataSource this.usePOST = instanceSettings.jsonData.usePOST || false; this.useCompression = instanceSettings.jsonData.useCompression || false; this.adHocValuesQuery = instanceSettings.jsonData.adHocValuesQuery || ''; + this.adHocHideTableNames = instanceSettings.jsonData.adHocHideTableNames || false; this.compressionType = instanceSettings.jsonData.compressionType || ''; this.defaultDatabase = instanceSettings.jsonData.defaultDatabase || ''; this.xHeaderUser = instanceSettings.jsonData.xHeaderUser || ''; diff --git a/src/spec/datasource.jest.ts b/src/spec/datasource.jest.ts index 198117296..986e7bf42 100644 --- a/src/spec/datasource.jest.ts +++ b/src/spec/datasource.jest.ts @@ -172,7 +172,7 @@ describe('clickhouse sql series:', () => { let adhocCtrl = new AdhocCtrl({ defaultDatabase: 'default' }); it('should be inited', function () { expect(adhocCtrl.query).toBe( - "SELECT database, table, name, type FROM system.columns WHERE database = 'default' AND database NOT IN ('system','INFORMATION_SCHEMA','information_schema') ORDER BY database, table" + "SELECT database, table, name, type FROM system.columns WHERE database = 'default' ORDER BY database, table" ); expect(adhocCtrl.datasource.defaultDatabase).toBe('default'); }); diff --git a/src/types/types.ts b/src/types/types.ts index 59856d132..231e758e6 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -75,7 +75,8 @@ export interface CHDataSourceOptions extends DataSourceJsonData { defaultTimeStamp64_3?: string; defaultTimeStamp64_6?: string; defaultTimeStamp64_9?: string; - adHocValuesQuery: string; + adHocValuesQuery?: string; + adHocHideTableNames?: boolean; contextWindowSize?: string; useWindowFuncForMacros?: boolean; } diff --git a/src/views/ConfigEditor/ConfigEditor.tsx b/src/views/ConfigEditor/ConfigEditor.tsx index 83e9813e9..738c53297 100644 --- a/src/views/ConfigEditor/ConfigEditor.tsx +++ b/src/views/ConfigEditor/ConfigEditor.tsx @@ -42,7 +42,7 @@ export function ConfigEditor(props: Props) { const onSwitchToggle = ( key: keyof Pick< CHDataSourceOptions, - 'useYandexCloudAuthorization' | 'addCorsHeader' | 'usePOST' | 'useCompression' | 'xClickHouseSSLCertificateAuth' + 'useYandexCloudAuthorization' | 'addCorsHeader' | 'usePOST' | 'useCompression' | 'xClickHouseSSLCertificateAuth' | 'adHocHideTableNames' >, value: boolean ) => { @@ -228,6 +228,14 @@ export function ConfigEditor(props: Props) { /> + + onSwitchToggle('adHocHideTableNames', e.currentTarget.checked)} + /> + );