Merge pull request #698 from Altinity/issue-456

Add `adhoc hide table names` connection settings option,

CHANGELOG.md

@@ -2,11 +2,12 @@
 ## Enhancements:
 * Add using window functions instead of `runningDifference` and `neighbor` for macros, to avoid `allow_deprecated_error_prone_window_functions`, fix https://github.com/Altinity/clickhouse-grafana/issues/572
 * Add public coverage report summary, fix https://github.com/Altinity/clickhouse-grafana/issues/660
-* Add support DateTime(timezone) types to Annotations query, fix https://github.com/Altinity/clickhouse-grafana/issues/642
+* Add support `DateTime(timezone)` types to Annotations query, fix https://github.com/Altinity/clickhouse-grafana/issues/642
 * Add single stat panel with categories, fix https://github.com/Altinity/clickhouse-grafana/issues/403
 * Add log context windows size to connection settings, fix https://github.com/Altinity/clickhouse-grafana/issues/657
 * Add `X-ClickHouse-SSL-Certificate-Auth` support, fix https://github.com/Altinity/clickhouse-grafana/issues/580
 * Add `$columnsMs` macro, fix https://github.com/Altinity/clickhouse-grafana/issues/430
+* Add `adhoc hide table names` connection settings option, fix https://github.com/Altinity/clickhouse-grafana/issues/456
 
 ## Fixes:
 * Add transposed table example, fix https://github.com/Altinity/clickhouse-grafana/issues/404 

docker/grafana/dashboards/adhoc_hide_table_names_issue_456.json

@@ -0,0 +1,176 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 43,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "vertamedia-clickhouse-datasource",
+        "uid": "P94617B551CAC0F6A"
+      },
+      "description": "reproduce https://github.com/Altinity/clickhouse-grafana/issues/456",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.4.0",
+      "targets": [
+        {
+          "adHocFilters": [],
+          "adHocValuesQuery": "",
+          "add_metadata": true,
+          "contextWindowSize": "10",
+          "database": "default",
+          "datasource": {
+            "type": "vertamedia-clickhouse-datasource",
+            "uid": "P94617B551CAC0F6A"
+          },
+          "dateTimeColDataType": "event_time",
+          "dateTimeType": "DATETIME",
+          "editorMode": "sql",
+          "extrapolate": true,
+          "format": "time_series",
+          "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t",
+          "interval": "",
+          "intervalFactor": 1,
+          "query": "SELECT $timeSeries as t, service_name, count() FROM $table WHERE $timeFilter GROUP BY t, service_name ORDER BY t",
+          "rawQuery": " /* grafana dashboard=AdHoc with Hiding table name, user=0 */\n\nSELECT\n    (intDiv(toUInt32(event_time), 20) * 20) * 1000 as t,\n    service_name,\n    count()\nFROM default.test_grafana\n\nWHERE\n    event_time >= toDateTime(1735113709) AND event_time <= toDateTime(1735135309)\n    AND service_name = 'service990'\nGROUP BY\n    t,\n    service_name\nORDER BY t\n",
+          "refId": "A",
+          "round": "0s",
+          "showFormattedSQL": true,
+          "skip_comments": true,
+          "table": "test_grafana",
+          "useWindowFuncForMacros": true
+        }
+      ],
+      "title": "AdHoc with Hide Table name",
+      "type": "timeseries"
+    }
+  ],
+  "preload": false,
+  "schemaVersion": 40,
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "baseFilters": [],
+        "datasource": {
+          "type": "vertamedia-clickhouse-datasource",
+          "uid": "P94617B551CAC0F6A"
+        },
+        "filters": [
+          {
+            "condition": "",
+            "key": "service_name",
+            "keyLabel": "service_name",
+            "operator": "=",
+            "value": "mysql",
+            "valueLabels": [
+              "mysql"
+            ]
+          }
+        ],
+        "name": "adhoc_hide_tables",
+        "type": "adhoc"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "AdHoc with Hiding table name",
+  "uid": "adhoc_hide_table_names",
+  "version": 1,
+  "weekStart": ""
+}

docker/grafana/provisioning/datasources/clickhouse-hide-adhoc-tables.yaml

@@ -0,0 +1,18 @@
+apiVersion: 1
+
+datasources:
+  - name: clickhouse-hide-adhoc-tables
+    type: vertamedia-clickhouse-datasource
+    access: proxy
+    url: http://clickhouse:8123
+    basicAuth: true
+    secureJsonData:
+      basicAuthPassword: ""
+    basicAuthUser: "default"
+    editable: false
+    jsonData:
+      addCorsHeader: true
+      usePOST: true
+      useCompression: true
+      compressionType: gzip
+      adHocHideTableNames: true

src/datasource/adhoc.ts

@@ -17,7 +17,7 @@ export default class AdHocFilter {
     this.adHocValuesQuery = datasource.adHocValuesQuery;
     let filter = queryFilter;
     if (datasource.defaultDatabase.length > 0) {
-      filter = "database = '" + datasource.defaultDatabase + "' AND " + queryFilter;
+      filter = "database = '" + datasource.defaultDatabase + "'";
     }
     this.query = columnsQuery.replace('{filter}', filter);
   }
@@ -47,22 +47,28 @@ export default class AdHocFilter {
       const databasePrefix = this.datasource.defaultDatabase.length === 0 ? item.database + '.' : '';
       const text: string = databasePrefix + item.table + '.' + item.name;
 
-      this.tagKeys.push({ text: text, value: text });
-
+      if (!this.datasource.adHocHideTableNames) {
+        this.tagKeys.push({ text: text, value: text });
+      }
       if (item.type.slice(0, 4) === 'Enum') {
         const regexEnum = /'(?:[^']+|'')+'/gim;
-        const options = item.type.match(regexEnum) || [];
-
-        if (options.length > 0) {
-          this.tagValues[text] = options.map((o: any) => ({ text: o, value: o }));
-          this.tagValues[item.name] = this.tagValues[text];
+        const enumValues = item.type.match(regexEnum) || [];
+
+        if (enumValues.length > 0) {
+          if (!this.datasource.adHocHideTableNames) {
+            this.tagValues[text] = enumValues.map((o: any) => ({ text: o, value: o }));
+          }
+          if (!this.tagValues[item.name]) {
+            this.tagValues[item.name] = this.tagValues[text];
+          } else {
+            this.tagValues[item.name].combine(this.tagValues[text]);
+          }
         }
       }
-
       columnNames[item.name] = true;
     });
 
-    // Store unique column names with wildcard table
+    // Store unique column names without table name
     Object.keys(columnNames).forEach((columnName) => {
       this.tagKeys.push({ text: columnName, value: columnName });
     });
@@ -73,9 +79,50 @@ export default class AdHocFilter {
   // GetTagValues returns column values according to passed options
   // Values for fields with Enum type were already fetched in GetTagKeys func and stored in `tagValues`
   // Values for fields which not represented on `tagValues` get from ClickHouse and cached on `tagValues`
-  GetTagValues(options) {
+  async GetTagValues(options) {
     // Determine which query to use initially
     const initialQuery = this.adHocValuesQuery || DEFAULT_VALUES_QUERY;
+    // Function to build the query
+    let database: string, table: string, field: string;
+    const buildQuery = (queryTemplate: string) =>
+      queryTemplate.replace('{field}', field).replace('{database}', database).replace('{table}', table);
+
+    if (this.datasource.adHocHideTableNames) {
+      // @todo could be very slow
+      const allTablesColumnSQL = "SELECT name,database,table FROM system.columns WHERE name='" + options.key + "'";
+      let allValuesSQL: string[] = [];
+      let isGetAllValuesOK: boolean = await this.datasource
+        .metricFindQuery(allTablesColumnSQL)
+        .then((response: any) => {
+          allValuesSQL = response.map((item: any) => {
+            field = item.name;
+            database = item.database;
+            table = item.table;
+            return buildQuery("(" + initialQuery + ")");
+          });
+          return true;
+        })
+        .catch((error: any) => {
+          console.error(error);
+          return false;
+        });
+
+      if (!isGetAllValuesOK) {
+        return [];
+      }
+      return this.datasource
+        .metricFindQuery(allValuesSQL.join(" UNION ALL "))
+        .then((response: any) => {
+          // Process and cache the response
+          this.tagValues[options.key] = this.processTagValuesResponse(response);
+          return this.tagValues[options.key];
+        })
+        .catch((error: any) => {
+          this.tagValues[options.key] = [];
+          console.error(error);
+          return this.tagValues[options.key];
+        });
+    }
 
     // If the tag values are already cached, return them immediately
     if (Object.prototype.hasOwnProperty.call(this.tagValues, options.key)) {
@@ -88,17 +135,13 @@ export default class AdHocFilter {
     }
 
     // Destructure key items based on their length
-    let database, table, field;
     if (keyItems.length === 3) {
       [database, table, field] = keyItems;
     } else {
       database = this.datasource.defaultDatabase;
       [table, field] = keyItems;
     }
 
-    // Function to build the query
-    const buildQuery = (queryTemplate) =>
-      queryTemplate.replace('{field}', field).replace('{database}', database).replace('{table}', table);
 
     // Execute the initial query
     return this.datasource

src/datasource/datasource.ts

@@ -50,6 +50,7 @@ export class CHDataSource
   useCompression: boolean;
   compressionType: string;
   adHocValuesQuery: string;
+  adHocHideTableNames: boolean;
   uid: string;
 
   constructor(instanceSettings: DataSourceInstanceSettings<CHDataSourceOptions>) {
@@ -62,6 +63,7 @@ export class CHDataSource
     this.usePOST = instanceSettings.jsonData.usePOST || false;
     this.useCompression = instanceSettings.jsonData.useCompression || false;
     this.adHocValuesQuery = instanceSettings.jsonData.adHocValuesQuery || '';
+    this.adHocHideTableNames = instanceSettings.jsonData.adHocHideTableNames || false;
     this.compressionType = instanceSettings.jsonData.compressionType || '';
     this.defaultDatabase = instanceSettings.jsonData.defaultDatabase || '';
     this.xHeaderUser = instanceSettings.jsonData.xHeaderUser || '';

src/spec/datasource.jest.ts

@@ -172,7 +172,7 @@ describe('clickhouse sql series:', () => {
     let adhocCtrl = new AdhocCtrl({ defaultDatabase: 'default' });
     it('should be inited', function () {
       expect(adhocCtrl.query).toBe(
-        "SELECT database, table, name, type FROM system.columns WHERE database = 'default' AND database NOT IN ('system','INFORMATION_SCHEMA','information_schema') ORDER BY database, table"
+        "SELECT database, table, name, type FROM system.columns WHERE database = 'default' ORDER BY database, table"
       );
       expect(adhocCtrl.datasource.defaultDatabase).toBe('default');
     });

src/types/types.ts

@@ -75,7 +75,8 @@ export interface CHDataSourceOptions extends DataSourceJsonData {
   defaultTimeStamp64_3?: string;
   defaultTimeStamp64_6?: string;
   defaultTimeStamp64_9?: string;
-  adHocValuesQuery: string;
+  adHocValuesQuery?: string;
+  adHocHideTableNames?: boolean;
   contextWindowSize?: string;
   useWindowFuncForMacros?: boolean;
 }

src/views/ConfigEditor/ConfigEditor.tsx

@@ -42,7 +42,7 @@ export function ConfigEditor(props: Props) {
   const onSwitchToggle = (
     key: keyof Pick<
       CHDataSourceOptions,
-      'useYandexCloudAuthorization' | 'addCorsHeader' | 'usePOST' | 'useCompression' | 'xClickHouseSSLCertificateAuth'
+      'useYandexCloudAuthorization' | 'addCorsHeader' | 'usePOST' | 'useCompression' | 'xClickHouseSSLCertificateAuth' | 'adHocHideTableNames'
     >,
     value: boolean
   ) => {
@@ -228,6 +228,14 @@ export function ConfigEditor(props: Props) {
             />
           </div>
         </InlineField>
+        <InlineField label="Hide table names in adhoc filters" labelWidth={32} tooltip="Applicable if you want adhoc with short filed names">
+          <InlineSwitch
+            data-test-id="adhoc-hide-table-names"
+            id="adhoc"
+            value={jsonData.adHocHideTableNames || false}
+            onChange={(e) => onSwitchToggle('adHocHideTableNames', e.currentTarget.checked)}
+          />
+        </InlineField>
       </div>
     </>
   );