Skip to content

Commit 12ec0e8

Browse files
caoxing9caoxing9
authored
perf: improve search index speed (#1193)
1 parent 975f4d0 commit 12ec0e8

File tree

7 files changed

+306
-57
lines changed

7 files changed

+306
-57
lines changed

‎apps/nestjs-backend/src/db-provider/db.provider.interface.ts

+6-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import type { DriverClient, IFilter, ISortItem } from '@teable/core';
22
import type { Prisma } from '@teable/db-main-prisma';
3-
import type { IAggregationField } from '@teable/openapi';
3+
import type { IAggregationField, ISearchIndexByQueryRo } from '@teable/openapi';
44
import type { Knex } from 'knex';
55
import type { IFieldInstance } from '../features/field/model/factory';
66
import type { DateFieldDto } from '../features/field/model/field-dto/date-field.dto';
@@ -136,9 +136,12 @@ export interface IDbProvider {
136136

137137
searchIndexQuery(
138138
originQueryBuilder: Knex.QueryBuilder,
139+
dbTableName: string,
139140
searchField: IFieldInstance[],
140-
searchValue: string,
141-
dbTableName: string
141+
searchIndexRo: Partial<ISearchIndexByQueryRo>,
142+
baseSortIndex?: string,
143+
setFilterQuery?: (qb: Knex.QueryBuilder) => void,
144+
setSortQuery?: (qb: Knex.QueryBuilder) => void
142145
): Knex.QueryBuilder;
143146

144147
searchCountQuery(

‎apps/nestjs-backend/src/db-provider/postgres.provider.ts

+14-9
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import { Logger } from '@nestjs/common';
33
import type { IFilter, ISortItem } from '@teable/core';
44
import { DriverClient } from '@teable/core';
55
import type { PrismaClient } from '@teable/db-main-prisma';
6-
import type { IAggregationField } from '@teable/openapi';
6+
import type { IAggregationField, ISearchIndexByQueryRo } from '@teable/openapi';
77
import type { Knex } from 'knex';
88
import type { IFieldInstance } from '../features/field/model/factory';
99
import type { SchemaType } from '../features/field/util';
@@ -23,7 +23,7 @@ import { FilterQueryPostgres } from './filter-query/postgres/filter-query.postgr
2323
import type { IGroupQueryExtra, IGroupQueryInterface } from './group-query/group-query.interface';
2424
import { GroupQueryPostgres } from './group-query/group-query.postgres';
2525
import { SearchQueryAbstract } from './search-query/abstract';
26-
import { SearchQueryPostgres } from './search-query/search-query.postgres';
26+
import { SearchQueryBuilder, SearchQueryPostgres } from './search-query/search-query.postgres';
2727
import { SortQueryPostgres } from './sort-query/postgres/sort-query.postgres';
2828
import type { ISortQueryInterface } from './sort-query/sort-query.interface';
2929

@@ -331,17 +331,22 @@ export class PostgresProvider implements IDbProvider {
331331

332332
searchIndexQuery(
333333
originQueryBuilder: Knex.QueryBuilder,
334+
dbTableName: string,
334335
searchField: IFieldInstance[],
335-
searchValue: string,
336-
dbTableName: string
336+
searchIndexRo: ISearchIndexByQueryRo,
337+
baseSortIndex?: string,
338+
setFilterQuery?: (qb: Knex.QueryBuilder) => void,
339+
setSortQuery?: (qb: Knex.QueryBuilder) => void
337340
) {
338-
return SearchQueryAbstract.buildSearchIndexQuery(
339-
SearchQueryPostgres,
341+
return new SearchQueryBuilder(
340342
originQueryBuilder,
343+
dbTableName,
341344
searchField,
342-
searchValue,
343-
dbTableName
344-
);
345+
searchIndexRo,
346+
baseSortIndex,
347+
setFilterQuery,
348+
setSortQuery
349+
).getSearchIndexQuery();
345350
}
346351

347352
shareFilterCollaboratorsQuery(

‎apps/nestjs-backend/src/db-provider/search-query/search-query.postgres.ts

+155-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
1-
import type { IDateFieldOptions } from '@teable/core';
2-
import type { Knex } from 'knex';
1+
import { CellValueType, type IDateFieldOptions } from '@teable/core';
2+
import type { ISearchIndexByQueryRo } from '@teable/openapi';
3+
import { type Knex } from 'knex';
34
import { get } from 'lodash';
45
import type { IFieldInstance } from '../../features/field/model/factory';
56
import { SearchQueryAbstract } from './abstract';
@@ -217,3 +218,155 @@ export class SearchQueryPostgres extends SearchQueryAbstract {
217218
.toQuery();
218219
}
219220
}
221+
222+
export class SearchQueryBuilder {
223+
constructor(
224+
public queryBuilder: Knex.QueryBuilder,
225+
public dbTableName: string,
226+
public searchField: IFieldInstance[],
227+
public searchIndexRo: ISearchIndexByQueryRo,
228+
public baseSortIndex?: string,
229+
public setFilterQuery?: (qb: Knex.QueryBuilder) => void,
230+
public setSortQuery?: (qb: Knex.QueryBuilder) => void
231+
) {
232+
this.queryBuilder = queryBuilder;
233+
this.dbTableName = dbTableName;
234+
this.searchField = searchField;
235+
this.baseSortIndex = baseSortIndex;
236+
this.searchIndexRo = searchIndexRo;
237+
this.setFilterQuery = setFilterQuery;
238+
this.setSortQuery = setSortQuery;
239+
}
240+
241+
getSearchQuery() {
242+
const { queryBuilder, searchIndexRo, searchField } = this;
243+
const { search } = searchIndexRo;
244+
const searchValue = search?.[0];
245+
246+
if (!search || !searchField?.length || !searchValue) {
247+
return queryBuilder;
248+
}
249+
250+
return searchField.map((field) => {
251+
const searchQueryBuilder = new SearchQueryPostgres(queryBuilder, field, searchValue);
252+
if (field.isMultipleCellValue) {
253+
switch (field.cellValueType) {
254+
case CellValueType.DateTime:
255+
return searchQueryBuilder.getMultipleDateSqlQuery();
256+
case CellValueType.Number:
257+
return searchQueryBuilder.getMultipleNumberSqlQuery();
258+
case CellValueType.String:
259+
if (field.isStructuredCellValue) {
260+
return searchQueryBuilder.getMultipleJsonSqlQuery();
261+
} else {
262+
return searchQueryBuilder.getMultipleTextSqlQuery();
263+
}
264+
}
265+
}
266+
267+
switch (field.cellValueType) {
268+
case CellValueType.DateTime:
269+
return searchQueryBuilder.getDateSqlQuery();
270+
case CellValueType.Number:
271+
return searchQueryBuilder.getNumberSqlQuery();
272+
case CellValueType.String:
273+
if (field.isStructuredCellValue) {
274+
return searchQueryBuilder.getJsonSqlQuery();
275+
} else {
276+
return searchQueryBuilder.getTextSqlQuery();
277+
}
278+
}
279+
});
280+
}
281+
282+
getCaseWhenSqlBy() {
283+
const { searchField, queryBuilder } = this;
284+
const searchQuerySql = this.getSearchQuery() as string[];
285+
return searchField.map(({ dbFieldName }, index) => {
286+
const knexInstance = queryBuilder.client;
287+
const searchSql = searchQuerySql[index];
288+
return knexInstance.raw(
289+
`
290+
CASE WHEN ${searchSql} THEN ? END
291+
`,
292+
[dbFieldName]
293+
);
294+
});
295+
}
296+
297+
getSearchIndexQuery() {
298+
const {
299+
queryBuilder,
300+
dbTableName,
301+
searchField,
302+
searchIndexRo,
303+
setFilterQuery,
304+
setSortQuery,
305+
baseSortIndex,
306+
} = this;
307+
308+
const { search, groupBy, orderBy } = searchIndexRo;
309+
const knexInstance = queryBuilder.client;
310+
311+
if (!search || !searchField.length) {
312+
return queryBuilder;
313+
}
314+
315+
const searchQuerySql = this.getSearchQuery() as string[];
316+
317+
const caseWhenQueryDbSql = this.getCaseWhenSqlBy() as string[];
318+
319+
queryBuilder.with('search_field_union_table', (qb) => {
320+
qb.select('*').select(
321+
knexInstance.raw(
322+
`array_remove(
323+
ARRAY [
324+
${caseWhenQueryDbSql.join(',')}
325+
],
326+
NULL
327+
) as matched_columns`
328+
)
329+
);
330+
331+
qb.from(dbTableName);
332+
333+
qb.where((subQb) => {
334+
subQb.where((orWhere) => {
335+
searchQuerySql.forEach((sql) => {
336+
orWhere.orWhereRaw(sql);
337+
});
338+
});
339+
if (this.searchIndexRo.filter && setFilterQuery) {
340+
subQb.andWhere((andQb) => {
341+
setFilterQuery?.(andQb);
342+
});
343+
}
344+
});
345+
346+
if (orderBy?.length || groupBy?.length) {
347+
setSortQuery?.(qb);
348+
}
349+
350+
baseSortIndex && qb.orderBy(baseSortIndex, 'asc');
351+
});
352+
353+
queryBuilder
354+
.select('*', 'matched_column')
355+
.select(
356+
knexInstance.raw(
357+
`CASE
358+
${searchField.map((field) => knexInstance.raw(`WHEN matched_column = '${field.dbFieldName}' THEN ?`, [field.id])).join(' ')}
359+
END AS "fieldId"`
360+
)
361+
)
362+
.fromRaw(
363+
`
364+
"search_field_union_table",
365+
LATERAL unnest(matched_columns) AS matched_column
366+
`
367+
)
368+
.whereRaw(`array_length(matched_columns, 1) > 0`);
369+
370+
return queryBuilder;
371+
}
372+
}

‎apps/nestjs-backend/src/db-provider/search-query/search-query.sqlite.ts

+96-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
import type { IDateFieldOptions } from '@teable/core';
1+
import { CellValueType, type IDateFieldOptions } from '@teable/core';
2+
import type { ISearchIndexByQueryRo } from '@teable/openapi';
23
import type { Knex } from 'knex';
34
import { get } from 'lodash';
45
import type { IFieldInstance } from '../../features/field/model/factory';
@@ -211,3 +212,97 @@ export class SearchQuerySqlite extends SearchQueryAbstract {
211212
.toQuery();
212213
}
213214
}
215+
216+
export class SearchQueryBuilder {
217+
constructor(
218+
public queryBuilder: Knex.QueryBuilder,
219+
public dbTableName: string,
220+
public searchField: IFieldInstance[],
221+
public searchIndexRo: ISearchIndexByQueryRo,
222+
public baseSortIndex?: string,
223+
public setFilterQuery?: (qb: Knex.QueryBuilder) => void,
224+
public setSortQuery?: (qb: Knex.QueryBuilder) => void
225+
) {
226+
this.queryBuilder = queryBuilder;
227+
this.dbTableName = dbTableName;
228+
this.searchField = searchField;
229+
this.baseSortIndex = baseSortIndex;
230+
this.searchIndexRo = searchIndexRo;
231+
this.setFilterQuery = setFilterQuery;
232+
this.setSortQuery = setSortQuery;
233+
}
234+
235+
getSearchIndexQuery() {
236+
const {
237+
queryBuilder,
238+
searchIndexRo,
239+
dbTableName,
240+
searchField,
241+
baseSortIndex,
242+
setFilterQuery,
243+
setSortQuery,
244+
} = this;
245+
const { search, take, skip, filter, orderBy, groupBy } = searchIndexRo;
246+
const knexInstance = queryBuilder.client;
247+
248+
if (!search || !searchField?.length) {
249+
return queryBuilder;
250+
}
251+
252+
queryBuilder.with('search_field_union_table', (qb) => {
253+
for (let index = 0; index < searchField.length; index++) {
254+
const currentWhereRaw = searchField[index];
255+
const dbFieldName = searchField[index].dbFieldName;
256+
257+
// boolean field or new field which does not support search should be skipped
258+
if (!currentWhereRaw || !dbFieldName) {
259+
continue;
260+
}
261+
262+
if (index === 0) {
263+
qb.select('*', knexInstance.raw(`? as matched_column`, [dbFieldName]))
264+
.whereRaw(`${currentWhereRaw}`)
265+
.from(dbTableName);
266+
} else {
267+
qb.unionAll(function () {
268+
this.select('*', knexInstance.raw(`? as matched_column`, [dbFieldName]))
269+
.whereRaw(`${currentWhereRaw}`)
270+
.from(dbTableName);
271+
});
272+
}
273+
}
274+
});
275+
276+
queryBuilder
277+
.select('__id', '__auto_number', 'matched_column')
278+
.select(
279+
knexInstance.raw(
280+
`CASE
281+
${searchField.map((field) => `WHEN matched_column = '${field.dbFieldName}' THEN '${field.id}'`).join(' ')}
282+
END AS "fieldId"`
283+
)
284+
)
285+
.from('search_field_union_table');
286+
287+
if (orderBy?.length || groupBy?.length) {
288+
setSortQuery?.(queryBuilder);
289+
}
290+
291+
if (filter) {
292+
setFilterQuery?.(queryBuilder);
293+
}
294+
295+
baseSortIndex && queryBuilder.orderBy(baseSortIndex, 'asc');
296+
297+
const cases = searchField.map((field, index) => {
298+
return knexInstance.raw(`CASE WHEN ?? = ? THEN ? END`, [
299+
'matched_column',
300+
field.dbFieldName,
301+
index + 1,
302+
]);
303+
});
304+
cases.length && queryBuilder.orderByRaw(cases.join(','));
305+
306+
return queryBuilder;
307+
}
308+
}

‎apps/nestjs-backend/src/db-provider/sqlite.provider.ts

+14-10
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import { Logger } from '@nestjs/common';
33
import type { IFilter, ISortItem } from '@teable/core';
44
import { DriverClient } from '@teable/core';
55
import type { PrismaClient } from '@teable/db-main-prisma';
6-
import type { IAggregationField } from '@teable/openapi';
6+
import type { IAggregationField, ISearchIndexByQueryRo } from '@teable/openapi';
77
import type { Knex } from 'knex';
88
import type { IFieldInstance } from '../features/field/model/factory';
99
import type { SchemaType } from '../features/field/util';
@@ -24,7 +24,7 @@ import type { IGroupQueryExtra, IGroupQueryInterface } from './group-query/group
2424
import { GroupQuerySqlite } from './group-query/group-query.sqlite';
2525
import { SearchQueryAbstract } from './search-query/abstract';
2626
import { getOffset } from './search-query/get-offset';
27-
import { SearchQuerySqlite } from './search-query/search-query.sqlite';
27+
import { SearchQueryBuilder, SearchQuerySqlite } from './search-query/search-query.sqlite';
2828
import type { ISortQueryInterface } from './sort-query/sort-query.interface';
2929
import { SortQuerySqlite } from './sort-query/sqlite/sort-query.sqlite';
3030

@@ -285,19 +285,23 @@ export class SqliteProvider implements IDbProvider {
285285

286286
searchIndexQuery(
287287
originQueryBuilder: Knex.QueryBuilder,
288+
dbTableName: string,
288289
searchField: IFieldInstance[],
289-
searchValue: string,
290-
dbTableName: string
290+
searchIndexRo: ISearchIndexByQueryRo,
291+
baseSortIndex?: string,
292+
setFilterQuery?: (qb: Knex.QueryBuilder) => void,
293+
setSortQuery?: (qb: Knex.QueryBuilder) => void
291294
) {
292-
return SearchQueryAbstract.buildSearchIndexQuery(
293-
SearchQuerySqlite,
295+
return new SearchQueryBuilder(
294296
originQueryBuilder,
297+
dbTableName,
295298
searchField,
296-
searchValue,
297-
dbTableName
298-
);
299+
searchIndexRo,
300+
baseSortIndex,
301+
setFilterQuery,
302+
setSortQuery
303+
).getSearchIndexQuery();
299304
}
300-
301305
shareFilterCollaboratorsQuery(
302306
originQueryBuilder: Knex.QueryBuilder,
303307
dbFieldName: string,

0 commit comments

Comments
 (0)