Allow manual placement of ad-hoc filters (#1488)

Author: jkeljo

README.md

@@ -268,6 +268,41 @@ For example, if `clickhouse_adhoc_query` is set to `SELECT DISTINCT
 machine_name FROM mgbench.logs1` you would be able to select which machine
 names are filtered for in the dashboard.
 
+#### Manual Ad Hoc Filter Placement with `$__adHocFilters`
+
+By default, ad-hoc filters are automatically applied to queries by detecting the
+target table using SQL parsing. However, for queries that use CTEs or ClickHouse-specific
+syntax like `INTERVAL` or aggregate functions with parameters, the automatic
+detection may fail. In these cases, you can manually specify where to apply
+ad-hoc filters using the `$__adHocFilters('table_name')` macro.
+
+This macro expands to the ClickHouse `additional_table_filters` setting with the
+currently active ad-hoc filters. It should be placed in the `SETTINGS` clause of
+your query.
+
+Example:
+
+```sql
+SELECT *
+FROM (
+  SELECT * FROM my_complex_table
+  WHERE complicated_condition
+) AS result
+SETTINGS $__adHocFilters('my_complex_table')
+```
+
+When ad-hoc filters are active (e.g., `status = 'active'` and `region = 'us-west'`),
+this expands to:
+
+```sql
+SELECT *
+FROM (
+  SELECT * FROM my_complex_table
+  WHERE complicated_condition
+) AS result
+SETTINGS additional_table_filters={'my_complex_table': 'status = \'active\' AND region = \'us-west\''}
+```
+
 ## Learn more
 
 - Add [Annotations](https://grafana.com/docs/grafana/latest/dashboards/annotations/).

src/ch-parser/pluginMacros.ts

@@ -92,4 +92,11 @@ export const pluginMacros: PluginMacro[] = [
       'Replaced by the first parameter when the template variable in the second parameter does not select every value. Replaced by 1=1 when the template variable selects every value',
     example: 'condition or 1=1',
   },
+  {
+    name: '$__adHocFilters',
+    isFunction: true,
+    documentation:
+      'Manually applies ad-hoc filters to a specific table. Useful for complex queries where automatic filter detection fails. Use in SETTINGS clause to specify the target table for ad-hoc filters',
+    example: "additional_table_filters={'table_name': 'column = \\'value\\' AND column2 = \\'value2\\'}",
+  },
 ];

src/data/CHDatasource.test.ts

@@ -148,6 +148,85 @@ describe('ClickHouseDatasource', () => {
       // Verify that the final query contains the ad-hoc filters
       expect(result.rawSql).toEqual(sqlWithAdHocFilters);
     });
+
+    it('should expand $__adHocFilters macro with single quotes', async () => {
+      const query = {
+        rawSql: "SELECT * FROM complex_table settings $__adHocFilters('my_table')",
+        editorType: EditorType.SQL,
+      } as CHQuery;
+
+      const adHocFilters = [
+        { key: 'key', operator: '=', value: 'val' },
+        { key: 'keyNum', operator: '=', value: '123' },
+      ];
+
+      const spyOnReplace = jest.spyOn(templateSrvMock, 'replace').mockImplementation((x) => x);
+      const spyOnGetVars = jest.spyOn(templateSrvMock, 'getVariables').mockImplementation(() => []);
+
+      const result = createInstance({}).applyTemplateVariables(query, {}, adHocFilters);
+
+      expect(spyOnReplace).toHaveBeenCalled();
+      expect(spyOnGetVars).toHaveBeenCalled();
+      expect(result.rawSql).toEqual(
+        "SELECT * FROM complex_table settings additional_table_filters={'my_table': ' key = \\'val\\' AND keyNum = \\'123\\' '}"
+      );
+    });
+
+    it('should expand $__adHocFilters macro with double quotes', async () => {
+      const query = {
+        rawSql: 'SELECT * FROM complex_table settings $__adHocFilters("my_table")',
+        editorType: EditorType.SQL,
+      } as CHQuery;
+
+      const adHocFilters = [{ key: 'key', operator: '=', value: 'val' }];
+
+      const spyOnReplace = jest.spyOn(templateSrvMock, 'replace').mockImplementation((x) => x);
+      const spyOnGetVars = jest.spyOn(templateSrvMock, 'getVariables').mockImplementation(() => []);
+
+      const result = createInstance({}).applyTemplateVariables(query, {}, adHocFilters);
+
+      expect(spyOnReplace).toHaveBeenCalled();
+      expect(spyOnGetVars).toHaveBeenCalled();
+      expect(result.rawSql).toEqual(
+        "SELECT * FROM complex_table settings additional_table_filters={'my_table': ' key = \\'val\\' '}"
+      );
+    });
+
+    it('should expand $__adHocFilters macro to empty object when no filters are present', async () => {
+      const query = {
+        rawSql: "SELECT * FROM complex_table settings $__adHocFilters('my_table')",
+        editorType: EditorType.SQL,
+      } as CHQuery;
+
+      const spyOnReplace = jest.spyOn(templateSrvMock, 'replace').mockImplementation((x) => x);
+      const spyOnGetVars = jest.spyOn(templateSrvMock, 'getVariables').mockImplementation(() => []);
+
+      const result = createInstance({}).applyTemplateVariables(query, {}, []);
+
+      expect(spyOnReplace).toHaveBeenCalled();
+      expect(spyOnGetVars).toHaveBeenCalled();
+      expect(result.rawSql).toEqual('SELECT * FROM complex_table settings additional_table_filters={}');
+    });
+
+    it('should handle $__adHocFilters macro with spaces', async () => {
+      const query = {
+        rawSql: "SELECT * FROM complex_table settings $__adHocFilters(  'my_table'  )",
+        editorType: EditorType.SQL,
+      } as CHQuery;
+
+      const adHocFilters = [{ key: 'key', operator: '=', value: 'val' }];
+
+      const spyOnReplace = jest.spyOn(templateSrvMock, 'replace').mockImplementation((x) => x);
+      const spyOnGetVars = jest.spyOn(templateSrvMock, 'getVariables').mockImplementation(() => []);
+
+      const result = createInstance({}).applyTemplateVariables(query, {}, adHocFilters);
+
+      expect(spyOnReplace).toHaveBeenCalled();
+      expect(spyOnGetVars).toHaveBeenCalled();
+      expect(result.rawSql).toEqual(
+        "SELECT * FROM complex_table settings additional_table_filters={'my_table': ' key = \\'val\\' '}"
+      );
+    });
   });
 
   describe('Tag Keys', () => {

src/data/CHDatasource.ts

@@ -248,7 +248,16 @@ export class Datasource
           `Unable to apply ad hoc filters. Upgrade ClickHouse to >=${this.adHocCHVerReq.major}.${this.adHocCHVerReq.minor} or remove ad hoc filters for the dashboard.`
         );
       }
-      rawQuery = this.adHocFilter.apply(rawQuery, filters);
+      // Check if query contains $__adHocFilters macro
+      const hasMacro = /\$__adHocFilters\s*\(\s*['"](.+?)['"]\s*\)/.test(rawQuery);
+
+      // Apply $__adHocFilters macro before automatic filter application
+      rawQuery = this.applyAdHocFiltersMacro(rawQuery, filters);
+
+      // Only apply automatic filters if the macro was not used
+      if (!hasMacro) {
+        rawQuery = this.adHocFilter.apply(rawQuery, filters);
+      }
     }
     this.skipAdHocFilter = false;
 
@@ -287,6 +296,23 @@ export class Datasource
     return rawQuery;
   }
 
+  applyAdHocFiltersMacro(rawQuery: string, filters: AdHocVariableFilter[]): string {
+    if (!rawQuery) {
+      return rawQuery;
+    }
+
+    // Match $__adHocFilters('table_name') or $__adHocFilters("table_name")
+    const regex = /\$__adHocFilters\s*\(\s*['"](.+?)['"]\s*\)/g;
+
+    return rawQuery.replace(regex, (match, tableName) => {
+      const filterStr = this.adHocFilter.buildFilterString(filters);
+      if (filterStr === '') {
+        return 'additional_table_filters={}';
+      }
+      return `additional_table_filters={'${tableName}': '${filterStr}'}`;
+    });
+  }
+
   // Support filtering by field value in Explore
   modifyQuery(query: CHQuery, action: QueryFixAction): CHQuery {
     if (query.editorType !== EditorType.Builder || !action.options || !action.options.key || !action.options.value) {

src/data/adHocFilter.test.ts

@@ -253,6 +253,52 @@ describe('AdHocManager', () => {
     expect(result).toContain('ResourceAttributes.cloud.region');
   });
 
+  describe('buildFilterString', () => {
+    it('builds filter string with single filter', () => {
+      const ahm = new AdHocFilter();
+      const result = ahm.buildFilterString([{ key: 'key', operator: '=', value: 'val' }] as AdHocVariableFilter[]);
+      expect(result).toEqual(" key = \\'val\\' ");
+    });
+
+    it('builds filter string with multiple filters', () => {
+      const ahm = new AdHocFilter();
+      const result = ahm.buildFilterString([
+        { key: 'key', operator: '=', value: 'val' },
+        { key: 'keyNum', operator: '=', value: '123' },
+      ] as AdHocVariableFilter[]);
+      expect(result).toEqual(" key = \\'val\\' AND keyNum = \\'123\\' ");
+    });
+
+    it('returns empty string with no filters', () => {
+      const ahm = new AdHocFilter();
+      const result = ahm.buildFilterString([]);
+      expect(result).toEqual('');
+    });
+
+    it('builds filter string with regex operators', () => {
+      const ahm = new AdHocFilter();
+      const result = ahm.buildFilterString([{ key: 'key', operator: '=~', value: 'val' }] as AdHocVariableFilter[]);
+      expect(result).toEqual(" key ILIKE \\'val\\' ");
+    });
+
+    it('builds filter string with IN operator', () => {
+      const ahm = new AdHocFilter();
+      const result = ahm.buildFilterString([
+        { key: 'key', operator: 'IN', value: "'val1', 'val2'" },
+      ] as AdHocVariableFilter[]);
+      expect(result).toEqual(" key IN (\\'val1\\', \\'val2\\') ");
+    });
+
+    it('ignores invalid filters', () => {
+      const ahm = new AdHocFilter();
+      const result = ahm.buildFilterString([
+        { key: 'key', operator: '=', value: 'val' },
+        { key: '', operator: '=', value: 'val' } as any,
+        { key: 'key2', operator: '=', value: 'val2' },
+      ] as AdHocVariableFilter[]);
+      expect(result).toEqual(" key = \\'val\\' AND key2 = \\'val2\\' ");
+    });
+  });
   it('should apply ad hoc filter with . in column name', () => {
     const ahm = new AdHocFilter();
     const val = ahm.apply('SELECT stuff FROM foo', [

src/data/adHocFilter.ts

@@ -11,6 +11,32 @@ export class AdHocFilter {
     }
   }
 
+  buildFilterString(adHocFilters: AdHocVariableFilter[]): string {
+    if (!adHocFilters || adHocFilters.length === 0) {
+      return '';
+    }
+
+    const validFilters = adHocFilters.filter((filter: AdHocVariableFilter) => {
+      const valid = isValid(filter);
+      if (!valid) {
+        console.warn('Invalid adhoc filter will be ignored:', filter);
+      }
+      return valid;
+    });
+
+    const filters = validFilters
+      .map((f, i) => {
+        const key = escapeKey(f.key);
+        const value = escapeValueBasedOnOperator(f.value, f.operator);
+        const condition = i !== validFilters.length - 1 ? (f.condition ? f.condition : 'AND') : '';
+        const operator = convertOperatorToClickHouseOperator(f.operator);
+        return ` ${key} ${operator} ${value} ${condition}`;
+      })
+      .join('');
+
+    return filters;
+  }
+
   apply(sql: string, adHocFilters: AdHocVariableFilter[]): string {
     if (sql === '' || !adHocFilters || adHocFilters.length === 0) {
       return sql;
@@ -29,22 +55,7 @@ export class AdHocFilter {
       return sql;
     }
 
-    const filters = adHocFilters
-      .filter((filter: AdHocVariableFilter) => {
-        const valid = isValid(filter);
-        if (!valid) {
-          console.warn('Invalid adhoc filter will be ignored:', filter);
-        }
-        return valid;
-      })
-      .map((f, i) => {
-        const key = escapeKey(f.key);
-        const value = escapeValueBasedOnOperator(f.value, f.operator);
-        const condition = i !== adHocFilters.length - 1 ? (f.condition ? f.condition : 'AND') : '';
-        const operator = convertOperatorToClickHouseOperator(f.operator);
-        return ` ${key} ${operator} ${value} ${condition}`;
-      })
-      .join('');
+    const filters = this.buildFilterString(adHocFilters);
 
     if (filters === '') {
       return sql;
@@ -56,7 +67,7 @@ export class AdHocFilter {
 }
 
 function isValid(filter: AdHocVariableFilter): boolean {
-  return filter.key !== undefined && filter.operator !== undefined && filter.value !== undefined;
+  return filter.key !== undefined && filter.key !== '' && filter.operator !== undefined && filter.value !== undefined;
 }
 
 function escapeKey(s: string): string {