Skip to content

Add macros $__adhoc_column #1491

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Smelentyev wants to merge 4 commits into
base: main
Choose a base branch
from
+147 −13
Open

Add macros $__adhoc_column #1491

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f0d670a
Add macros $__adhoc_column
Smelentyev Dec 26, 2025
49a8f23
fix cspell checks
Smelentyev Jan 6, 2026
79854d4
fix cspell check
Smelentyev Jan 12, 2026
d87ce01
Merge branch 'main' into fix-adhoc-long-query
Smelentyev Jan 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,41 @@ For example, if `clickhouse_adhoc_query` is set to `SELECT DISTINCT
machine_name FROM mgbench.logs1` you would be able to select which machine
names are filtered for in the dashboard.

##### Using the `$__adhoc_column` macro

The `$__adhoc_column` macro allows you to write a single query template that
works for all columns in a table. This is especially useful for large tables
where querying all columns at once would be too expensive.

The macro will be replaced with the actual column name when fetching values
for each ad-hoc filter. You can combine it with time filter macros to limit
the data being scanned.

For example:
```sql
SELECT DISTINCT $__adhoc_column
FROM database.table
WHERE $__timeFilter(timestamp)
LIMIT 1000
```

This query will:
1. Extract the table name and fetch all available columns for ad-hoc filtering
2. When a user selects a value for a specific column (e.g., `hostname`), the
query becomes: `SELECT DISTINCT column FROM database.table
WHERE $__timeFilter(timestamp) LIMIT 1000`
3. Apply the time range from the dashboard's time picker to limit the scan

You can also use other macros and ClickHouse functions:
```sql
SELECT DISTINCT $__adhoc_column
FROM database.table
WHERE $__timeFilter(timestamp)
AND event != ''
ORDER BY $__adhoc_column
LIMIT 500
```

## Learn more

- Add [Annotations](https://grafana.com/docs/grafana/latest/dashboards/annotations/).
Expand Down
1 change: 1 addition & 0 deletions cspell.config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
"endregion",
"eror",
"errorsource",
"EXCP",
"fixedstring",
"fromtime",
"goproxy",
Expand Down
7 changes: 7 additions & 0 deletions src/ch-parser/pluginMacros.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,4 +92,11 @@ export const pluginMacros: PluginMacro[] = [
'Replaced by the first parameter when the template variable in the second parameter does not select every value. Replaced by 1=1 when the template variable selects every value',
example: 'condition or 1=1',
},
{
name: '$__adhoc_column',
isFunction: false,
documentation:
'Used in ad-hoc query ($clickhouse_adhoc_query). Replaced by the current column name when fetching tag values for ad-hoc filters',
example: 'hostname',
},
];
40 changes: 40 additions & 0 deletions src/data/CHDatasource.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,46 @@ describe('ClickHouseDatasource', () => {
expect(values).toEqual([{ text: 'foo' }]);
});

it('should replace $__adhoc_column macro when fetching tag values', async () => {
const spyOnReplace = jest
.spyOn(templateSrvMock, 'replace')
.mockImplementation(() => 'SELECT DISTINCT $__adhoc_column FROM foo.bar WHERE timestamp > now() - INTERVAL 1 DAY LIMIT 1000');
const ds = cloneDeep(mockDatasource);
const frame = arrayToDataFrame([{ hostname: 'server1' }, { hostname: 'server2' }]);
const spyOnQuery = jest.spyOn(ds, 'query').mockImplementation((_request) => of({ data: [frame] }));

const values = await ds.getTagValues({ key: 'hostname' });

expect(spyOnReplace).toHaveBeenCalled();
const expected = { rawSql: 'SELECT DISTINCT hostname FROM foo.bar WHERE timestamp > now() - INTERVAL 1 DAY LIMIT 1000' };

expect(spyOnQuery).toHaveBeenCalledWith(
expect.objectContaining({ targets: expect.arrayContaining([expect.objectContaining(expected)]) })
);

expect(values).toEqual([{ text: 'server1' }, { text: 'server2' }]);
});

it('should replace $__adhoc_column with time filter macro', async () => {
const spyOnReplace = jest
.spyOn(templateSrvMock, 'replace')
.mockImplementation(() => 'SELECT DISTINCT $__adhoc_column FROM db.table WHERE $__timeFilter(timestamp) LIMIT 500');
const ds = cloneDeep(mockDatasource);
const frame = arrayToDataFrame([{ event: 'EXCP' }, { event: 'CALL' }]);
const spyOnQuery = jest.spyOn(ds, 'query').mockImplementation((_request) => of({ data: [frame] }));

const values = await ds.getTagValues({ key: 'event' });

expect(spyOnReplace).toHaveBeenCalled();
const expected = { rawSql: 'SELECT DISTINCT event FROM db.table WHERE $__timeFilter(timestamp) LIMIT 500' };

expect(spyOnQuery).toHaveBeenCalledWith(
expect.objectContaining({ targets: expect.arrayContaining([expect.objectContaining(expected)]) })
);

expect(values).toEqual([{ text: 'EXCP' }, { text: 'CALL' }]);
});

it('should Fetch Tag Values from Schema with . in column name', async () => {
const spyOnReplace = jest.spyOn(templateSrvMock, 'replace').mockImplementation(() => '$clickhouse_adhoc_query');
const ds = cloneDeep(mockDatasource);
Expand Down
77 changes: 64 additions & 13 deletions src/data/CHDatasource.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
DataSourceInstanceSettings,
DataSourceWithLogsContextSupport,
DataSourceWithSupplementaryQueriesSupport,
Field,
getTimeZone,
getTimeZoneInfo,
LogRowContextOptions,
Expand Down Expand Up @@ -849,6 +850,15 @@ export class Datasource
return this.fetchTagValuesFromSchema(key);
}

private fieldValuesToMetricFindValues(field: Field): MetricFindValue[] {
// Convert to string to avoid https://github.com/grafana/grafana/issues/12209
return field.values
.filter((value) => value !== null)
.map((value) => {
return { text: String(value) };
});
}

private async fetchTagValuesFromSchema(key: string): Promise<MetricFindValue[]> {
const { from } = this.getTagSource();
const [table, ...colParts] = key.split('.');
Expand All @@ -860,24 +870,32 @@ export class Datasource
return [];
}
const field = frame.fields[0];
// Convert to string to avoid https://github.com/grafana/grafana/issues/12209
return field.values
.filter((value) => value !== null)
.map((value) => {
return { text: String(value) };
});
return this.fieldValuesToMetricFindValues(field);
}

private async fetchTagValuesFromQuery(key: string): Promise<MetricFindValue[]> {
const tagSource = this.getTagSource();

// Check if the query contains the $__adhoc_column macro
if (tagSource.source && tagSource.source.includes('$__adhoc_column')) {
// Replace the macro with the actual column name
const queryWithColumn = tagSource.source.replace(/\$__adhoc_column/g, key);
this.skipAdHocFilter = true;
const frame = await this.runQuery({ rawSql: queryWithColumn });

if (frame.fields?.length === 0) {
return [];
}

const field = frame.fields[0];
return this.fieldValuesToMetricFindValues(field);
}

// Fallback to the original behavior
const { frame } = await this.fetchTags();
const field = frame.fields.find((f) => f.name === key);
if (field) {
// Convert to string to avoid https://github.com/grafana/grafana/issues/12209
return field.values
.filter((value) => value !== null)
.map((value) => {
return { text: String(value) };
});
return this.fieldValuesToMetricFindValues(field);
}
return [];
}
Expand All @@ -893,13 +911,46 @@ export class Datasource
}

if (tagSource.type === TagType.query) {
this.adHocFilter.setTargetTableFromQuery(tagSource.source);
// Check if the query contains the $__adhoc_column macro
if (tagSource.source.includes('$__adhoc_column')) {
// Extract table name from the query and get column list from system.columns
const tableName = this.extractTableNameFromQuery(tagSource.source);
if (tableName) {
this.adHocFilter.setTargetTableFromQuery(tagSource.source.replace(/\$__adhoc_column/g, '*'));

// Parse database.table format
const parts = tableName.split('.');
let query: string;
if (parts.length === 2) {
const [db, table] = parts;
query = `SELECT name, type, table FROM system.columns WHERE database = '${db}' AND table = '${table}'`;
} else {
query = `SELECT name, type, table FROM system.columns WHERE table = '${tableName}'`;
}
const results = await this.runQuery({ rawSql: query });
return { type: TagType.schema, frame: results };
}
} else {
this.adHocFilter.setTargetTableFromQuery(tagSource.source);
}
}

const results = await this.runQuery({ rawSql: tagSource.source });
return { type: tagSource.type, frame: results };
}

private extractTableNameFromQuery(query: string): string | null {
// Try to extract table name from FROM clause
// Supports formats: FROM table, FROM database.table, FROM "database"."table"
const fromMatch = query.match(/FROM\s+(?:"?(\w+)"?\.)?"?(\w+)"?/i);
if (fromMatch) {
const database = fromMatch[1];
const table = fromMatch[2];
return database ? `${database}.${table}` : table;
}
return null;
}

private getTagSource() {
// @todo https://github.com/grafana/grafana/issues/13109
const ADHOC_VAR = '$clickhouse_adhoc_query';
Expand Down
Loading