diff --git a/README.md b/README.md index f874b33f0..f326d1ee8 100644 --- a/README.md +++ b/README.md @@ -268,6 +268,41 @@ For example, if `clickhouse_adhoc_query` is set to `SELECT DISTINCT machine_name FROM mgbench.logs1` you would be able to select which machine names are filtered for in the dashboard. +##### Using the `$__adhoc_column` macro + +The `$__adhoc_column` macro allows you to write a single query template that +works for all columns in a table. This is especially useful for large tables +where querying all columns at once would be too expensive. + +The macro will be replaced with the actual column name when fetching values +for each ad-hoc filter. You can combine it with time filter macros to limit +the data being scanned. + +For example: +```sql +SELECT DISTINCT $__adhoc_column +FROM database.table +WHERE $__timeFilter(timestamp) +LIMIT 1000 +``` + +This query will: +1. Extract the table name and fetch all available columns for ad-hoc filtering +2. When a user selects a value for a specific column (e.g., `hostname`), the + query becomes: `SELECT DISTINCT column FROM database.table + WHERE $__timeFilter(timestamp) LIMIT 1000` +3. Apply the time range from the dashboard's time picker to limit the scan + +You can also use other macros and ClickHouse functions: +```sql +SELECT DISTINCT $__adhoc_column +FROM database.table +WHERE $__timeFilter(timestamp) + AND event != '' +ORDER BY $__adhoc_column +LIMIT 500 +``` + ## Learn more - Add [Annotations](https://grafana.com/docs/grafana/latest/dashboards/annotations/). diff --git a/cspell.config.json b/cspell.config.json index 33483451f..faf4dc156 100644 --- a/cspell.config.json +++ b/cspell.config.json @@ -31,6 +31,7 @@ "endregion", "eror", "errorsource", + "EXCP", "fixedstring", "fromtime", "goproxy", diff --git a/src/ch-parser/pluginMacros.ts b/src/ch-parser/pluginMacros.ts index 630eabcc6..f75b2768e 100644 --- a/src/ch-parser/pluginMacros.ts +++ b/src/ch-parser/pluginMacros.ts @@ -92,4 +92,11 @@ export const pluginMacros: PluginMacro[] = [ 'Replaced by the first parameter when the template variable in the second parameter does not select every value. Replaced by 1=1 when the template variable selects every value', example: 'condition or 1=1', }, + { + name: '$__adhoc_column', + isFunction: false, + documentation: + 'Used in ad-hoc query ($clickhouse_adhoc_query). Replaced by the current column name when fetching tag values for ad-hoc filters', + example: 'hostname', + }, ]; diff --git a/src/data/CHDatasource.test.ts b/src/data/CHDatasource.test.ts index ddfa56427..f36a22874 100644 --- a/src/data/CHDatasource.test.ts +++ b/src/data/CHDatasource.test.ts @@ -270,6 +270,46 @@ describe('ClickHouseDatasource', () => { expect(values).toEqual([{ text: 'foo' }]); }); + it('should replace $__adhoc_column macro when fetching tag values', async () => { + const spyOnReplace = jest + .spyOn(templateSrvMock, 'replace') + .mockImplementation(() => 'SELECT DISTINCT $__adhoc_column FROM foo.bar WHERE timestamp > now() - INTERVAL 1 DAY LIMIT 1000'); + const ds = cloneDeep(mockDatasource); + const frame = arrayToDataFrame([{ hostname: 'server1' }, { hostname: 'server2' }]); + const spyOnQuery = jest.spyOn(ds, 'query').mockImplementation((_request) => of({ data: [frame] })); + + const values = await ds.getTagValues({ key: 'hostname' }); + + expect(spyOnReplace).toHaveBeenCalled(); + const expected = { rawSql: 'SELECT DISTINCT hostname FROM foo.bar WHERE timestamp > now() - INTERVAL 1 DAY LIMIT 1000' }; + + expect(spyOnQuery).toHaveBeenCalledWith( + expect.objectContaining({ targets: expect.arrayContaining([expect.objectContaining(expected)]) }) + ); + + expect(values).toEqual([{ text: 'server1' }, { text: 'server2' }]); + }); + + it('should replace $__adhoc_column with time filter macro', async () => { + const spyOnReplace = jest + .spyOn(templateSrvMock, 'replace') + .mockImplementation(() => 'SELECT DISTINCT $__adhoc_column FROM db.table WHERE $__timeFilter(timestamp) LIMIT 500'); + const ds = cloneDeep(mockDatasource); + const frame = arrayToDataFrame([{ event: 'EXCP' }, { event: 'CALL' }]); + const spyOnQuery = jest.spyOn(ds, 'query').mockImplementation((_request) => of({ data: [frame] })); + + const values = await ds.getTagValues({ key: 'event' }); + + expect(spyOnReplace).toHaveBeenCalled(); + const expected = { rawSql: 'SELECT DISTINCT event FROM db.table WHERE $__timeFilter(timestamp) LIMIT 500' }; + + expect(spyOnQuery).toHaveBeenCalledWith( + expect.objectContaining({ targets: expect.arrayContaining([expect.objectContaining(expected)]) }) + ); + + expect(values).toEqual([{ text: 'EXCP' }, { text: 'CALL' }]); + }); + it('should Fetch Tag Values from Schema with . in column name', async () => { const spyOnReplace = jest.spyOn(templateSrvMock, 'replace').mockImplementation(() => '$clickhouse_adhoc_query'); const ds = cloneDeep(mockDatasource); diff --git a/src/data/CHDatasource.ts b/src/data/CHDatasource.ts index 4a963e0f3..74d087df8 100644 --- a/src/data/CHDatasource.ts +++ b/src/data/CHDatasource.ts @@ -7,6 +7,7 @@ import { DataSourceInstanceSettings, DataSourceWithLogsContextSupport, DataSourceWithSupplementaryQueriesSupport, + Field, getTimeZone, getTimeZoneInfo, LogRowContextOptions, @@ -849,6 +850,15 @@ export class Datasource return this.fetchTagValuesFromSchema(key); } + private fieldValuesToMetricFindValues(field: Field): MetricFindValue[] { + // Convert to string to avoid https://github.com/grafana/grafana/issues/12209 + return field.values + .filter((value) => value !== null) + .map((value) => { + return { text: String(value) }; + }); + } + private async fetchTagValuesFromSchema(key: string): Promise { const { from } = this.getTagSource(); const [table, ...colParts] = key.split('.'); @@ -860,24 +870,32 @@ export class Datasource return []; } const field = frame.fields[0]; - // Convert to string to avoid https://github.com/grafana/grafana/issues/12209 - return field.values - .filter((value) => value !== null) - .map((value) => { - return { text: String(value) }; - }); + return this.fieldValuesToMetricFindValues(field); } private async fetchTagValuesFromQuery(key: string): Promise { + const tagSource = this.getTagSource(); + + // Check if the query contains the $__adhoc_column macro + if (tagSource.source && tagSource.source.includes('$__adhoc_column')) { + // Replace the macro with the actual column name + const queryWithColumn = tagSource.source.replace(/\$__adhoc_column/g, key); + this.skipAdHocFilter = true; + const frame = await this.runQuery({ rawSql: queryWithColumn }); + + if (frame.fields?.length === 0) { + return []; + } + + const field = frame.fields[0]; + return this.fieldValuesToMetricFindValues(field); + } + + // Fallback to the original behavior const { frame } = await this.fetchTags(); const field = frame.fields.find((f) => f.name === key); if (field) { - // Convert to string to avoid https://github.com/grafana/grafana/issues/12209 - return field.values - .filter((value) => value !== null) - .map((value) => { - return { text: String(value) }; - }); + return this.fieldValuesToMetricFindValues(field); } return []; } @@ -893,13 +911,46 @@ export class Datasource } if (tagSource.type === TagType.query) { - this.adHocFilter.setTargetTableFromQuery(tagSource.source); + // Check if the query contains the $__adhoc_column macro + if (tagSource.source.includes('$__adhoc_column')) { + // Extract table name from the query and get column list from system.columns + const tableName = this.extractTableNameFromQuery(tagSource.source); + if (tableName) { + this.adHocFilter.setTargetTableFromQuery(tagSource.source.replace(/\$__adhoc_column/g, '*')); + + // Parse database.table format + const parts = tableName.split('.'); + let query: string; + if (parts.length === 2) { + const [db, table] = parts; + query = `SELECT name, type, table FROM system.columns WHERE database = '${db}' AND table = '${table}'`; + } else { + query = `SELECT name, type, table FROM system.columns WHERE table = '${tableName}'`; + } + const results = await this.runQuery({ rawSql: query }); + return { type: TagType.schema, frame: results }; + } + } else { + this.adHocFilter.setTargetTableFromQuery(tagSource.source); + } } const results = await this.runQuery({ rawSql: tagSource.source }); return { type: tagSource.type, frame: results }; } + private extractTableNameFromQuery(query: string): string | null { + // Try to extract table name from FROM clause + // Supports formats: FROM table, FROM database.table, FROM "database"."table" + const fromMatch = query.match(/FROM\s+(?:"?(\w+)"?\.)?"?(\w+)"?/i); + if (fromMatch) { + const database = fromMatch[1]; + const table = fromMatch[2]; + return database ? `${database}.${table}` : table; + } + return null; + } + private getTagSource() { // @todo https://github.com/grafana/grafana/issues/13109 const ADHOC_VAR = '$clickhouse_adhoc_query';