From d4953868367bac61ffb54a63671d061d3021c841 Mon Sep 17 00:00:00 2001 From: tea artist Date: Sat, 11 Jan 2025 22:27:53 +0800 Subject: [PATCH] feat: link field integrity check --- README.md | 8 + .../src/db-provider/postgres.provider.ts | 15 +- .../src/db-provider/sqlite.provider.ts | 16 +- .../src/features/calculation/batch.service.ts | 8 +- .../features/calculation/reference.service.ts | 2 +- .../features/integrity/foreign-key.service.ts | 175 +++++++++ .../features/integrity/integrity.module.ts | 4 +- .../features/integrity/link-field.service.ts | 339 ++++++++++++++++++ .../integrity/link-integrity.service.ts | 210 ++--------- .../nestjs-backend/test/integrity.e2e-spec.ts | 322 +++++++++++++++++ apps/nextjs-app/.env.development | 4 +- package.json | 2 +- packages/openapi/src/integrity/link-check.ts | 3 +- plugins/README.md | 230 +++++++++++- 14 files changed, 1144 insertions(+), 194 deletions(-) create mode 100644 apps/nestjs-backend/src/features/integrity/foreign-key.service.ts create mode 100644 apps/nestjs-backend/src/features/integrity/link-field.service.ts diff --git a/README.md b/README.md index 959acef0cc..38f0e52ddd 100644 --- a/README.md +++ b/README.md @@ -204,6 +204,14 @@ cd apps/nestjs-backend pnpm dev ``` +By default, the plugin development server is not started. To preview and develop plugins, run: +```sh +cd plugins +pnpm dev +``` +This will start the plugin development server on port 3002. + + ## Why Teable? No-code tools have significantly speed up how we get things done, allowing non-tech users to build amazing apps and changing the way many work and live. People like using spreadsheet-like UI to handle their data because it's easy, flexible, and great for team collaboration. They also prefer designing their app screens without being stuck with clunky templates. diff --git a/apps/nestjs-backend/src/db-provider/postgres.provider.ts b/apps/nestjs-backend/src/db-provider/postgres.provider.ts index 5b460d2fd4..91d2724e84 100644 --- a/apps/nestjs-backend/src/db-provider/postgres.provider.ts +++ b/apps/nestjs-backend/src/db-provider/postgres.provider.ts @@ -450,9 +450,22 @@ export class PostgresProvider implements IDbProvider { .select({ tableId: 'table_id', id: 'id', - type: 'type', name: 'name', + description: 'description', + notNull: 'not_null', + unique: 'unique', + isPrimary: 'is_primary', + dbFieldName: 'db_field_name', + isComputed: 'is_computed', + isPending: 'is_pending', + hasError: 'has_error', + dbFieldType: 'db_field_type', + isMultipleCellValue: 'is_multiple_cell_value', + isLookup: 'is_lookup', + lookupOptions: 'lookup_options', + type: 'type', options: 'options', + cellValueType: 'cell_value_type', }) .whereNull('deleted_time') .whereNull('is_lookup') diff --git a/apps/nestjs-backend/src/db-provider/sqlite.provider.ts b/apps/nestjs-backend/src/db-provider/sqlite.provider.ts index 58d855368a..703e2b93cc 100644 --- a/apps/nestjs-backend/src/db-provider/sqlite.provider.ts +++ b/apps/nestjs-backend/src/db-provider/sqlite.provider.ts @@ -406,10 +406,24 @@ export class SqliteProvider implements IDbProvider { optionsQuery(type: FieldType, optionsKey: string, value: string): string { return this.knex('field') .select({ + tableId: 'table_id', id: 'id', - type: 'type', name: 'name', + description: 'description', + notNull: 'not_null', + unique: 'unique', + isPrimary: 'is_primary', + dbFieldName: 'db_field_name', + isComputed: 'is_computed', + isPending: 'is_pending', + hasError: 'has_error', + dbFieldType: 'db_field_type', + isMultipleCellValue: 'is_multiple_cell_value', + isLookup: 'is_lookup', + lookupOptions: 'lookup_options', + type: 'type', options: 'options', + cellValueType: 'cell_value_type', }) .where('type', type) .whereNull('is_lookup') diff --git a/apps/nestjs-backend/src/features/calculation/batch.service.ts b/apps/nestjs-backend/src/features/calculation/batch.service.ts index df050bcb8f..da26caab41 100644 --- a/apps/nestjs-backend/src/features/calculation/batch.service.ts +++ b/apps/nestjs-backend/src/features/calculation/batch.service.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import { Injectable, Logger } from '@nestjs/common'; +import { BadRequestException, Injectable, Logger } from '@nestjs/common'; import type { IOtOperation } from '@teable/core'; import { IdPrefix, RecordOpBuilder } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; @@ -104,6 +104,12 @@ export class BatchService { ); const versionGroup = keyBy(raw, '__id'); + opsPair.map(([recordId]) => { + if (!versionGroup[recordId]) { + throw new BadRequestException(`Record ${recordId} not found in ${tableId}`); + } + }); + const opsData = this.buildRecordOpsData(opsPair, versionGroup); if (!opsData.length) return; diff --git a/apps/nestjs-backend/src/features/calculation/reference.service.ts b/apps/nestjs-backend/src/features/calculation/reference.service.ts index e4ed488716..733b9e67fc 100644 --- a/apps/nestjs-backend/src/features/calculation/reference.service.ts +++ b/apps/nestjs-backend/src/features/calculation/reference.service.ts @@ -617,7 +617,7 @@ export class ReferenceService { const result = dependenciesIndexed[v.id]; if (!result) { throw new InternalServerErrorException( - `Record not found for: ${JSON.stringify(v)}, fieldId: ${field.id}` + `Record not found for: ${JSON.stringify(v)}, fieldId: ${field.id}, when calculate ${JSON.stringify(recordItem.record.id)}` ); } return result; diff --git a/apps/nestjs-backend/src/features/integrity/foreign-key.service.ts b/apps/nestjs-backend/src/features/integrity/foreign-key.service.ts new file mode 100644 index 0000000000..07fce2a716 --- /dev/null +++ b/apps/nestjs-backend/src/features/integrity/foreign-key.service.ts @@ -0,0 +1,175 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { FieldType, type ILinkFieldOptions } from '@teable/core'; +import { Prisma, PrismaService } from '@teable/db-main-prisma'; +import { IntegrityIssueType, type IIntegrityIssue } from '@teable/openapi'; +import { Knex } from 'knex'; +import { InjectModel } from 'nest-knexjs'; +import type { LinkFieldDto } from '../field/model/field-dto/link-field.dto'; + +@Injectable() +export class ForeignKeyIntegrityService { + private readonly logger = new Logger(ForeignKeyIntegrityService.name); + + constructor( + private readonly prismaService: PrismaService, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex + ) {} + + async getIssues(tableId: string, field: LinkFieldDto): Promise { + const { foreignTableId, fkHostTableName, foreignKeyName, selfKeyName } = field.options; + const issues: IIntegrityIssue[] = []; + + const { name: selfTableName, dbTableName: selfTableDbTableName } = + await this.prismaService.tableMeta.findFirstOrThrow({ + where: { id: tableId, deletedTime: null }, + select: { name: true, dbTableName: true }, + }); + + const { name: foreignTableName, dbTableName: foreignTableDbTableName } = + await this.prismaService.tableMeta.findFirstOrThrow({ + where: { id: foreignTableId, deletedTime: null }, + select: { name: true, dbTableName: true }, + }); + + // Check self references + if (selfTableDbTableName !== fkHostTableName) { + const selfIssues = await this.checkInvalidReferences({ + fkHostTableName, + targetTableName: selfTableDbTableName, + keyName: selfKeyName, + field, + referencedTableName: selfTableName, + isSelfReference: true, + }); + issues.push(...selfIssues); + } + + // Check foreign references + if (foreignTableDbTableName !== fkHostTableName) { + const foreignIssues = await this.checkInvalidReferences({ + fkHostTableName, + targetTableName: foreignTableDbTableName, + keyName: foreignKeyName, + field, + referencedTableName: foreignTableName, + isSelfReference: false, + }); + issues.push(...foreignIssues); + } + + return issues; + } + + private async checkInvalidReferences({ + fkHostTableName, + targetTableName, + keyName, + field, + referencedTableName, + isSelfReference, + }: { + fkHostTableName: string; + targetTableName: string; + keyName: string; + field: { id: string; name: string }; + referencedTableName: string; + isSelfReference: boolean; + }): Promise { + const issues: IIntegrityIssue[] = []; + + const invalidQuery = this.knex(fkHostTableName) + .leftJoin(targetTableName, `${fkHostTableName}.${keyName}`, `${targetTableName}.__id`) + .whereNull(`${targetTableName}.__id`) + .count(`${fkHostTableName}.${keyName} as count`) + .first() + .toQuery(); + + try { + const invalidRefs = + await this.prismaService.$queryRawUnsafe<{ count: bigint }[]>(invalidQuery); + const refCount = Number(invalidRefs[0]?.count || 0); + + if (refCount > 0) { + const message = isSelfReference + ? `Found ${refCount} invalid self references in table ${referencedTableName}` + : `Found ${refCount} invalid foreign references to table ${referencedTableName}`; + + issues.push({ + type: IntegrityIssueType.MissingRecordReference, + message: `${message} (Field Name: ${field.name}, Field ID: ${field.id})`, + }); + } + } catch (error) { + if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === 'P2010') { + console.error('error ignored:', error); + } else { + throw error; + } + } + + return issues; + } + + async fix(_tableId: string, fieldId: string): Promise { + const field = await this.prismaService.field.findFirstOrThrow({ + where: { id: fieldId, type: FieldType.Link, isLookup: null, deletedTime: null }, + }); + + const options = JSON.parse(field.options as string) as ILinkFieldOptions; + const { foreignTableId, fkHostTableName, foreignKeyName, selfKeyName } = options; + const foreignTable = await this.prismaService.tableMeta.findFirstOrThrow({ + where: { id: foreignTableId, deletedTime: null }, + select: { id: true, name: true, dbTableName: true }, + }); + + let totalFixed = 0; + + // Fix invalid self references + if (fkHostTableName !== fkHostTableName) { + const selfDeleted = await this.deleteMissingReferences({ + fkHostTableName, + targetTableName: fkHostTableName, + keyName: selfKeyName, + }); + totalFixed += selfDeleted; + } + + // Fix invalid foreign references + if (foreignTable.dbTableName !== fkHostTableName) { + const foreignDeleted = await this.deleteMissingReferences({ + fkHostTableName, + targetTableName: foreignTable.dbTableName, + keyName: foreignKeyName, + }); + totalFixed += foreignDeleted; + } + + if (totalFixed > 0) { + return { + type: IntegrityIssueType.MissingRecordReference, + message: `Fixed ${totalFixed} invalid references and inconsistent links for link field (Field Name: ${field.name}, Field ID: ${field.id})`, + }; + } + } + + private async deleteMissingReferences({ + fkHostTableName, + targetTableName, + keyName, + }: { + fkHostTableName: string; + targetTableName: string; + keyName: string; + }) { + const deleteQuery = this.knex(fkHostTableName) + .whereNotExists( + this.knex + .select('__id') + .from(targetTableName) + .where('__id', this.knex.ref(`${fkHostTableName}.${keyName}`)) + ) + .delete() + .toQuery(); + return await this.prismaService.$executeRawUnsafe(deleteQuery); + } +} diff --git a/apps/nestjs-backend/src/features/integrity/integrity.module.ts b/apps/nestjs-backend/src/features/integrity/integrity.module.ts index a94b5a1ade..57e312f373 100644 --- a/apps/nestjs-backend/src/features/integrity/integrity.module.ts +++ b/apps/nestjs-backend/src/features/integrity/integrity.module.ts @@ -1,10 +1,12 @@ import { Module } from '@nestjs/common'; +import { ForeignKeyIntegrityService } from './foreign-key.service'; import { IntegrityController } from './integrity.controller'; +import { LinkFieldIntegrityService } from './link-field.service'; import { LinkIntegrityService } from './link-integrity.service'; @Module({ controllers: [IntegrityController], - providers: [LinkIntegrityService], + providers: [ForeignKeyIntegrityService, LinkFieldIntegrityService, LinkIntegrityService], exports: [LinkIntegrityService], }) export class IntegrityModule {} diff --git a/apps/nestjs-backend/src/features/integrity/link-field.service.ts b/apps/nestjs-backend/src/features/integrity/link-field.service.ts new file mode 100644 index 0000000000..47cd56b3a8 --- /dev/null +++ b/apps/nestjs-backend/src/features/integrity/link-field.service.ts @@ -0,0 +1,339 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { FieldType, type ILinkFieldOptions } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { IntegrityIssueType, type IIntegrityIssue } from '@teable/openapi'; +import { Knex } from 'knex'; +import { InjectModel } from 'nest-knexjs'; +import { createFieldInstanceByRaw } from '../field/model/factory'; +import type { LinkFieldDto } from '../field/model/field-dto/link-field.dto'; + +@Injectable() +export class LinkFieldIntegrityService { + private readonly logger = new Logger(LinkFieldIntegrityService.name); + + constructor( + private readonly prismaService: PrismaService, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex + ) {} + + async getIssues(tableId: string, field: LinkFieldDto): Promise { + const table = await this.prismaService.tableMeta.findFirstOrThrow({ + where: { id: tableId, deletedTime: null }, + select: { dbTableName: true }, + }); + const { fkHostTableName, foreignKeyName, selfKeyName } = field.options; + const inconsistentRecords = await this.checkLinks({ + dbTableName: table.dbTableName, + fkHostTableName, + selfKeyName, + foreignKeyName, + linkDbFieldName: field.dbFieldName, + isMultiValue: Boolean(field.isMultipleCellValue), + }); + + if (inconsistentRecords.length > 0) { + return [ + { + type: IntegrityIssueType.InvalidLinkReference, + message: `Found ${inconsistentRecords.length} inconsistent links in table ${fkHostTableName} (Field Name: ${field.name}, Field ID: ${field.id})`, + }, + ]; + } + + return []; + } + + private async checkLinks({ + dbTableName, + fkHostTableName, + selfKeyName, + foreignKeyName, + linkDbFieldName, + isMultiValue, + }: { + dbTableName: string; + fkHostTableName: string; + selfKeyName: string; + foreignKeyName: string; + linkDbFieldName: string; + isMultiValue: boolean; + }) { + if (isMultiValue) { + const fkGroupedQuery = this.knex(fkHostTableName) + .select({ + [selfKeyName]: selfKeyName, + fk_ids: this.knex.raw(`string_agg(??, ',' ORDER BY ??)`, [ + this.knex.ref(foreignKeyName), + this.knex.ref(foreignKeyName), + ]), + }) + .whereNotNull(selfKeyName) + .groupBy(selfKeyName) + .as('fk_grouped'); + const thisKnex = this.knex; + const query = this.knex(dbTableName) + .leftJoin(fkGroupedQuery, `${dbTableName}.__id`, `fk_grouped.${selfKeyName}`) + .select({ + id: '__id', + }) + .where(function () { + this.whereNull(`fk_grouped.${selfKeyName}`) + .whereNotNull(linkDbFieldName) + .orWhere(function () { + this.whereNotNull(linkDbFieldName).andWhereRaw( + `"fk_grouped".fk_ids != ( + SELECT string_agg(id, ',' ORDER BY id) + FROM ( + SELECT (link->>'id')::text as id + FROM jsonb_array_elements(??::jsonb) as link + ) t + )`, + [thisKnex.ref(linkDbFieldName)] + ); + }); + }) + .toQuery(); + + return await this.prismaService.$queryRawUnsafe< + { + id: string; + }[] + >(query); + } + + if (fkHostTableName === dbTableName) { + const query = this.knex(dbTableName) + .select({ + id: '__id', + }) + .where(function () { + this.whereNull(foreignKeyName) + .whereNotNull(linkDbFieldName) + .orWhere(function () { + this.whereNotNull(linkDbFieldName).andWhereRaw( + `("${linkDbFieldName}"->>'id')::text != "${foreignKeyName}"::text` + ); + }); + }) + .toQuery(); + + return await this.prismaService.$queryRawUnsafe< + { + id: string; + }[] + >(query); + } + + if (dbTableName === fkHostTableName) { + const query = this.knex(`${dbTableName} as t1`) + .select({ + id: 't1.__id', + }) + .leftJoin(`${dbTableName} as t2`, 't2.' + foreignKeyName, 't1.__id') + .where(function () { + this.whereNull('t2.' + foreignKeyName) + .whereNotNull('t1.' + linkDbFieldName) + .orWhere(function () { + this.whereNotNull('t1.' + linkDbFieldName).andWhereRaw( + `("t1"."${linkDbFieldName}"->>'id')::text != "t2"."${foreignKeyName}"::text` + ); + }); + }) + .toQuery(); + + return await this.prismaService.$queryRawUnsafe< + { + id: string; + }[] + >(query); + } + + const query = this.knex(`${dbTableName} as t1`) + .select({ + id: 't1.__id', + }) + .leftJoin(`${fkHostTableName} as t2`, 't2.' + selfKeyName, 't1.__id') + .where(function () { + this.whereNull('t2.' + foreignKeyName) + .whereNotNull('t1.' + linkDbFieldName) + .orWhere(function () { + this.whereNotNull('t1.' + linkDbFieldName).andWhereRaw( + `("t1"."${linkDbFieldName}"->>'id')::text != "t2"."${foreignKeyName}"::text` + ); + }); + }) + .toQuery(); + + return await this.prismaService.$queryRawUnsafe< + { + id: string; + }[] + >(query); + } + + private async fixLinks({ + recordIds, + dbTableName, + foreignDbTableName, + fkHostTableName, + lookupDbFieldName, + selfKeyName, + foreignKeyName, + linkDbFieldName, + isMultiValue, + }: { + recordIds: string[]; + dbTableName: string; + foreignDbTableName: string; + fkHostTableName: string; + lookupDbFieldName: string; + selfKeyName: string; + foreignKeyName: string; + linkDbFieldName: string; + isMultiValue: boolean; + }) { + if (isMultiValue) { + const query = this.knex(dbTableName) + .update({ + [linkDbFieldName]: this.knex + .select( + this.knex.raw("jsonb_agg(jsonb_build_object('id', ??, 'title', ??) ORDER BY ??)", [ + `fk.${foreignKeyName}`, + `ft.${lookupDbFieldName}`, + `fk.${foreignKeyName}`, + ]) + ) + .from(`${fkHostTableName} as fk`) + .join(`${foreignDbTableName} as ft`, `ft.__id`, `fk.${foreignKeyName}`) + .where('fk.' + selfKeyName, `${dbTableName}.__id`), + }) + .whereIn('__id', recordIds) + .toQuery(); + + return await this.prismaService.$executeRawUnsafe(query); + } + + if (fkHostTableName === dbTableName) { + // Handle self-referential single-value links + const query = this.knex(dbTableName) + .update({ + [linkDbFieldName]: this.knex.raw( + ` + CASE + WHEN ?? IS NULL THEN NULL + ELSE jsonb_build_object( + 'id', ??, + 'title', ?? + ) + END + `, + [foreignKeyName, foreignKeyName, lookupDbFieldName] + ), + }) + .whereIn('__id', recordIds) + .toQuery(); + + return await this.prismaService.$executeRawUnsafe(query); + } + + // Handle cross-table single-value links + const query = this.knex(dbTableName) + .update({ + [linkDbFieldName]: this.knex + .select( + this.knex.raw( + `CASE + WHEN t2.?? IS NULL THEN NULL + ELSE jsonb_build_object('id', t2.??, 'title', t2.??) + END`, + [foreignKeyName, foreignKeyName, lookupDbFieldName] + ) + ) + .from(`${fkHostTableName} as t2`) + .where(`t2.${foreignKeyName}`, `${dbTableName}.__id`) + .limit(1), + }) + .whereIn('__id', recordIds) + .toQuery(); + + return await this.prismaService.$executeRawUnsafe(query); + } + + private async checkAndFix(params: { + dbTableName: string; + foreignDbTableName: string; + fkHostTableName: string; + lookupDbFieldName: string; + foreignKeyName: string; + linkDbFieldName: string; + isMultiValue: boolean; + selfKeyName: string; + }) { + try { + const inconsistentRecords = await this.checkLinks(params); + + if (inconsistentRecords.length > 0) { + const recordIds = inconsistentRecords.map((record) => record.id); + const updatedCount = await this.fixLinks({ + ...params, + recordIds, + }); + this.logger.debug(`Updated ${updatedCount} records in ${params.dbTableName}`); + return updatedCount; + } + return 0; + } catch (error) { + this.logger.error('Error updating inconsistent links:', error); + throw error; + } + } + + async fix(tableId: string, fieldId: string): Promise { + const table = await this.prismaService.tableMeta.findFirstOrThrow({ + where: { id: tableId, deletedTime: null }, + select: { dbTableName: true }, + }); + + const field = await this.prismaService.field.findFirstOrThrow({ + where: { id: fieldId, type: FieldType.Link, isLookup: null, deletedTime: null }, + }); + + const linkField = createFieldInstanceByRaw(field) as LinkFieldDto; + + const lookupField = await this.prismaService.field.findFirstOrThrow({ + where: { id: linkField.options.lookupFieldId, deletedTime: null }, + select: { dbFieldName: true }, + }); + + const foreignTable = await this.prismaService.tableMeta.findFirstOrThrow({ + where: { id: linkField.options.foreignTableId, deletedTime: null }, + select: { dbTableName: true }, + }); + + const options = JSON.parse(field.options as string) as ILinkFieldOptions; + const { fkHostTableName, foreignKeyName, selfKeyName } = options; + + let totalFixed = 0; + + // Add table links fixing + const linksFixed = await this.checkAndFix({ + dbTableName: table.dbTableName, + foreignDbTableName: foreignTable.dbTableName, + fkHostTableName, + lookupDbFieldName: lookupField.dbFieldName, + foreignKeyName, + linkDbFieldName: linkField.dbFieldName, + isMultiValue: Boolean(linkField.isMultipleCellValue), + selfKeyName, + }); + + totalFixed += linksFixed; + + if (totalFixed > 0) { + return { + type: IntegrityIssueType.InvalidLinkReference, + message: `Fixed ${totalFixed} inconsistent links for link field (Field Name: ${field.name}, Field ID: ${field.id})`, + }; + } + } +} diff --git a/apps/nestjs-backend/src/features/integrity/link-integrity.service.ts b/apps/nestjs-backend/src/features/integrity/link-integrity.service.ts index 96d11a8eda..e821a22841 100644 --- a/apps/nestjs-backend/src/features/integrity/link-integrity.service.ts +++ b/apps/nestjs-backend/src/features/integrity/link-integrity.service.ts @@ -1,12 +1,14 @@ import { Injectable, Logger } from '@nestjs/common'; import { FieldType, type ILinkFieldOptions } from '@teable/core'; import type { Field } from '@teable/db-main-prisma'; -import { Prisma, PrismaService } from '@teable/db-main-prisma'; +import { PrismaService } from '@teable/db-main-prisma'; import { IntegrityIssueType, type IIntegrityCheckVo, type IIntegrityIssue } from '@teable/openapi'; -import { Knex } from 'knex'; -import { InjectModel } from 'nest-knexjs'; import { InjectDbProvider } from '../../db-provider/db.provider'; import { IDbProvider } from '../../db-provider/db.provider.interface'; +import { createFieldInstanceByRaw } from '../field/model/factory'; +import type { LinkFieldDto } from '../field/model/field-dto/link-field.dto'; +import { ForeignKeyIntegrityService } from './foreign-key.service'; +import { LinkFieldIntegrityService } from './link-field.service'; @Injectable() export class LinkIntegrityService { @@ -14,8 +16,9 @@ export class LinkIntegrityService { constructor( private readonly prismaService: PrismaService, - @InjectDbProvider() private readonly dbProvider: IDbProvider, - @InjectModel('CUSTOM_KNEX') private readonly knex: Knex + private readonly foreignKeyIntegrityService: ForeignKeyIntegrityService, + private readonly linkFieldIntegrityService: LinkFieldIntegrityService, + @InjectDbProvider() private readonly dbProvider: IDbProvider ) {} async linkIntegrityCheck(baseId: string): Promise { @@ -171,111 +174,19 @@ export class LinkIntegrityService { } if (foreignTable) { - const invalidReferences = await this.checkInvalidRecordReferences(table.id, field, options); + const linkField = createFieldInstanceByRaw(field) as LinkFieldDto; + const invalidReferences = await this.foreignKeyIntegrityService.getIssues( + table.id, + linkField + ); + const invalidLinks = await this.linkFieldIntegrityService.getIssues(table.id, linkField); if (invalidReferences.length > 0) { issues.push(...invalidReferences); } - } - } - - return issues; - } - - private async checkInvalidRecordReferences( - tableId: string, - field: { id: string; name: string }, - options: ILinkFieldOptions - ): Promise { - const { foreignTableId, fkHostTableName, foreignKeyName, selfKeyName } = options; - - const { name: selfTableName, dbTableName: selfTableDbTableName } = - await this.prismaService.tableMeta.findFirstOrThrow({ - where: { id: tableId, deletedTime: null }, - select: { name: true, dbTableName: true }, - }); - - const { name: foreignTableName, dbTableName: foreignTableDbTableName } = - await this.prismaService.tableMeta.findFirstOrThrow({ - where: { id: foreignTableId, deletedTime: null }, - select: { name: true, dbTableName: true }, - }); - - const issues: IIntegrityIssue[] = []; - - // Check self references - if (selfTableDbTableName !== fkHostTableName) { - const selfIssues = await this.checkInvalidReferences({ - fkHostTableName, - targetTableName: selfTableDbTableName, - keyName: selfKeyName, - field, - referencedTableName: selfTableName, - isSelfReference: true, - }); - issues.push(...selfIssues); - } - - // Check foreign references - if (foreignTableDbTableName !== fkHostTableName) { - const foreignIssues = await this.checkInvalidReferences({ - fkHostTableName, - targetTableName: foreignTableDbTableName, - keyName: foreignKeyName, - field, - referencedTableName: foreignTableName, - isSelfReference: false, - }); - issues.push(...foreignIssues); - } - - return issues; - } - - private async checkInvalidReferences({ - fkHostTableName, - targetTableName, - keyName, - field, - referencedTableName, - isSelfReference, - }: { - fkHostTableName: string; - targetTableName: string; - keyName: string; - field: { id: string; name: string }; - referencedTableName: string; - isSelfReference: boolean; - }): Promise { - const issues: IIntegrityIssue[] = []; - - const invalidQuery = this.knex(fkHostTableName) - .leftJoin(targetTableName, `${fkHostTableName}.${keyName}`, `${targetTableName}.__id`) - .whereNull(`${targetTableName}.__id`) - .count(`${fkHostTableName}.${keyName} as count`) - .first() - .toQuery(); - - try { - const invalidRefs = - await this.prismaService.$queryRawUnsafe<{ count: bigint }[]>(invalidQuery); - const refCount = Number(invalidRefs[0]?.count || 0); - - if (refCount > 0) { - const message = isSelfReference - ? `Found ${refCount} invalid self references in table ${referencedTableName}` - : `Found ${refCount} invalid foreign references to table ${referencedTableName}`; - - issues.push({ - type: IntegrityIssueType.InvalidRecordReference, - message: `${message} (Field Name: ${field.name}, Field ID: ${field.id})`, - }); - } - } catch (error) { - if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === 'P2010') { - console.error('error ignored:', error); - } else { - throw error; + if (invalidLinks.length > 0) { + issues.push(...invalidLinks); + } } } @@ -290,8 +201,16 @@ export class LinkIntegrityService { for (const issue of issues.issues) { // eslint-disable-next-line sonarjs/no-small-switch switch (issue.type) { - case IntegrityIssueType.InvalidRecordReference: { - const result = await this.fixInvalidRecordReferences(issues.tableId, issues.fieldId); + case IntegrityIssueType.MissingRecordReference: { + const result = await this.foreignKeyIntegrityService.fix( + issues.tableId, + issues.fieldId + ); + result && fixResults.push(result); + break; + } + case IntegrityIssueType.InvalidLinkReference: { + const result = await this.linkFieldIntegrityService.fix(issues.tableId, issues.fieldId); result && fixResults.push(result); break; } @@ -303,79 +222,4 @@ export class LinkIntegrityService { return fixResults; } - - async fixInvalidRecordReferences( - tableId: string, - fieldId: string - ): Promise { - const field = await this.prismaService.field.findFirstOrThrow({ - where: { id: fieldId, type: FieldType.Link, isLookup: null, deletedTime: null }, - }); - - const options = JSON.parse(field.options as string) as ILinkFieldOptions; - const { foreignTableId, fkHostTableName, foreignKeyName, selfKeyName } = options; - - const { dbTableName: selfTableDbTableName } = - await this.prismaService.tableMeta.findFirstOrThrow({ - where: { id: tableId, deletedTime: null }, - select: { dbTableName: true }, - }); - - const { dbTableName: foreignTableDbTableName } = - await this.prismaService.tableMeta.findFirstOrThrow({ - where: { id: foreignTableId, deletedTime: null }, - select: { dbTableName: true }, - }); - - let totalDeleted = 0; - - // Fix invalid self references - if (selfTableDbTableName !== fkHostTableName) { - const selfDeleted = await this.deleteInvalidReferences({ - fkHostTableName, - targetTableName: selfTableDbTableName, - keyName: selfKeyName, - }); - totalDeleted += selfDeleted; - } - - // Fix invalid foreign references - if (foreignTableDbTableName !== fkHostTableName) { - const foreignDeleted = await this.deleteInvalidReferences({ - fkHostTableName, - targetTableName: foreignTableDbTableName, - keyName: foreignKeyName, - }); - totalDeleted += foreignDeleted; - } - - if (totalDeleted > 0) { - return { - type: IntegrityIssueType.InvalidRecordReference, - message: `Fixed ${totalDeleted} invalid references for link field (Field Name: ${field.name}, Field ID: ${field.id})`, - }; - } - } - - private async deleteInvalidReferences({ - fkHostTableName, - targetTableName, - keyName, - }: { - fkHostTableName: string; - targetTableName: string; - keyName: string; - }) { - const deleteQuery = this.knex(fkHostTableName) - .whereNotExists( - this.knex - .select('__id') - .from(targetTableName) - .where('__id', this.knex.ref(`${fkHostTableName}.${keyName}`)) - ) - .delete() - .toQuery(); - - return await this.prismaService.$executeRawUnsafe(deleteQuery); - } } diff --git a/apps/nestjs-backend/test/integrity.e2e-spec.ts b/apps/nestjs-backend/test/integrity.e2e-spec.ts index 15d27b5483..444e6ca656 100644 --- a/apps/nestjs-backend/test/integrity.e2e-spec.ts +++ b/apps/nestjs-backend/test/integrity.e2e-spec.ts @@ -14,6 +14,10 @@ import { createBase, deleteBase, fixBaseIntegrity, + getRecord, + getRecords, + updateRecord, + updateRecords, } from '@teable/openapi'; import type { Knex } from 'knex'; import { @@ -162,5 +166,323 @@ describe('OpenAPI integrity (e2e)', () => { const integrity3 = await checkBaseIntegrity(baseId2); expect(integrity3.data.hasIssues).toEqual(false); }); + + it('should check integrity when a many-one link field cell value is more than foreignKey', async () => { + const linkFieldRo: IFieldRo = { + name: 'link field', + type: FieldType.Link, + options: { + baseId: baseId2, + relationship: Relationship.ManyOne, + foreignTableId: base2table2.id, + }, + }; + + const linkField = await createField(base2table1.id, linkFieldRo); + const symLinkField = await getField( + base2table2.id, + (linkField.options as ILinkFieldOptions).symmetricFieldId as string + ); + + expect((symLinkField.options as ILinkFieldOptions).baseId).toBeUndefined(); + + await updateRecords(base2table1.id, { + records: [ + { + id: base2table1.records[0].id, + fields: { + [base2table1.fields[0].name]: 'a1', + }, + }, + { + id: base2table1.records[1].id, + fields: { + [base2table1.fields[0].name]: 'a2', + }, + }, + ], + }); + + await updateRecord(base2table2.id, base2table2.records[0].id, { + record: { + fields: { + [base2table2.fields[0].name]: 'b1', + [symLinkField.name]: [ + { id: base2table1.records[0].id }, + { id: base2table1.records[1].id }, + ], + }, + }, + }); + + const integrity = await checkBaseIntegrity(baseId2); + expect(integrity.data.hasIssues).toEqual(false); + + // test multiple link + await executeKnex( + db(base2table2.dbTableName) + .where('__id', base2table2.records[0].id) + .update({ + [symLinkField.dbFieldName]: db.raw(`jsonb_set( + "${symLinkField.dbFieldName}", + '{0,id}', + '"xxx"' + )`), + }) + ); + + const record = await getRecord(base2table2.id, base2table2.records[0].id); + expect(record.data.fields[symLinkField.name]).toEqual([ + { id: 'xxx', title: 'a1' }, + { id: base2table1.records[1].id, title: 'a2' }, + ]); + + const integrity2 = await checkBaseIntegrity(baseId2); + expect(integrity2.data.hasIssues).toEqual(true); + expect(integrity2.data.linkFieldIssues.length).toEqual(1); + + await fixBaseIntegrity(baseId2); + + const integrity3 = await checkBaseIntegrity(baseId2); + expect(integrity3.data.hasIssues).toEqual(false); + + // test single link + await executeKnex( + db(base2table1.dbTableName) + .where('__id', base2table1.records[0].id) + .update({ + [linkField.dbFieldName]: db.raw(`jsonb_set( + "${linkField.dbFieldName}", + '{id}', + '"xxx"' + )`), + }) + ); + + const record2 = await getRecord(base2table1.id, base2table1.records[0].id); + expect(record2.data.fields[linkField.name]).toEqual({ id: 'xxx', title: 'b1' }); + + const integrity4 = await checkBaseIntegrity(baseId2); + expect(integrity4.data.hasIssues).toEqual(true); + + await fixBaseIntegrity(baseId2); + + const integrity5 = await checkBaseIntegrity(baseId2); + expect(integrity5.data.hasIssues).toEqual(false); + }); + + it('should check integrity when a one-one link field cell value is more than foreignKey', async () => { + const linkFieldRo: IFieldRo = { + name: 'link field', + type: FieldType.Link, + options: { + baseId: baseId2, + relationship: Relationship.OneOne, + foreignTableId: base2table2.id, + }, + }; + + const linkField = await createField(base2table1.id, linkFieldRo); + const symLinkField = await getField( + base2table2.id, + (linkField.options as ILinkFieldOptions).symmetricFieldId as string + ); + + expect((symLinkField.options as ILinkFieldOptions).baseId).toBeUndefined(); + + await updateRecords(base2table1.id, { + records: [ + { + id: base2table1.records[0].id, + fields: { + [base2table1.fields[0].name]: 'a1', + }, + }, + { + id: base2table1.records[1].id, + fields: { + [base2table1.fields[0].name]: 'a2', + }, + }, + ], + }); + + await updateRecords(base2table2.id, { + records: [ + { + id: base2table2.records[0].id, + fields: { + [base2table2.fields[0].name]: 'b1', + [symLinkField.name]: { id: base2table1.records[0].id }, + }, + }, + { + id: base2table2.records[1].id, + fields: { + [base2table2.fields[0].name]: 'b2', + [symLinkField.name]: { id: base2table1.records[1].id }, + }, + }, + ], + }); + + const integrity = await checkBaseIntegrity(baseId2); + expect(integrity.data.hasIssues).toEqual(false); + + // test multiple link + await executeKnex( + db(base2table2.dbTableName) + .whereIn('__id', [base2table2.records[0].id, base2table2.records[1].id]) + .update({ + [symLinkField.dbFieldName]: db.raw(`jsonb_set( + "${symLinkField.dbFieldName}", + '{id}', + '"xxx"' + )`), + }) + ); + + const records = await getRecords(base2table2.id); + expect(records.data.records[0].fields[symLinkField.name]).toEqual({ id: 'xxx', title: 'a1' }); + expect(records.data.records[1].fields[symLinkField.name]).toEqual({ id: 'xxx', title: 'a2' }); + + const integrity2 = await checkBaseIntegrity(baseId2); + expect(integrity2.data.hasIssues).toEqual(true); + expect(integrity2.data.linkFieldIssues.length).toEqual(1); + + await fixBaseIntegrity(baseId2); + + const integrity3 = await checkBaseIntegrity(baseId2); + expect(integrity3.data.hasIssues).toEqual(false); + + // test single link + await executeKnex( + db(base2table1.dbTableName) + .whereIn('__id', [base2table1.records[0].id, base2table1.records[1].id]) + .update({ + [linkField.dbFieldName]: db.raw(`jsonb_set( + "${linkField.dbFieldName}", + '{id}', + '"xxx"' + )`), + }) + ); + + const records2 = await getRecords(base2table1.id); + expect(records2.data.records[0].fields[linkField.name]).toEqual({ id: 'xxx', title: 'b1' }); + expect(records2.data.records[1].fields[linkField.name]).toEqual({ id: 'xxx', title: 'b2' }); + + const integrity4 = await checkBaseIntegrity(baseId2); + expect(integrity4.data.hasIssues).toEqual(true); + + await fixBaseIntegrity(baseId2); + + const integrity5 = await checkBaseIntegrity(baseId2); + expect(integrity5.data.hasIssues).toEqual(false); + }); + + it('should check integrity when a many-many link field cell value is more than foreignKey', async () => { + const linkFieldRo: IFieldRo = { + name: 'link field', + type: FieldType.Link, + options: { + baseId: baseId2, + relationship: Relationship.ManyMany, + foreignTableId: base2table2.id, + }, + }; + + const linkField = await createField(base2table1.id, linkFieldRo); + const symLinkField = await getField( + base2table2.id, + (linkField.options as ILinkFieldOptions).symmetricFieldId as string + ); + + expect((symLinkField.options as ILinkFieldOptions).baseId).toBeUndefined(); + + await updateRecords(base2table1.id, { + records: [ + { + id: base2table1.records[0].id, + fields: { + [base2table1.fields[0].name]: 'a1', + }, + }, + { + id: base2table1.records[1].id, + fields: { + [base2table1.fields[0].name]: 'a2', + }, + }, + ], + }); + + await updateRecord(base2table2.id, base2table2.records[0].id, { + record: { + fields: { + [base2table2.fields[0].name]: 'b1', + [symLinkField.name]: [ + { id: base2table1.records[0].id }, + { id: base2table1.records[1].id }, + ], + }, + }, + }); + + const integrity = await checkBaseIntegrity(baseId2); + expect(integrity.data.hasIssues).toEqual(false); + + // test multiple link + await executeKnex( + db(base2table2.dbTableName) + .where('__id', base2table2.records[0].id) + .update({ + [symLinkField.dbFieldName]: db.raw(`jsonb_set( + "${symLinkField.dbFieldName}", + '{0,id}', + '"xxx"' + )`), + }) + ); + + const record = await getRecord(base2table2.id, base2table2.records[0].id); + expect(record.data.fields[symLinkField.name]).toEqual([ + { id: 'xxx', title: 'a1' }, + { id: base2table1.records[1].id, title: 'a2' }, + ]); + + const integrity2 = await checkBaseIntegrity(baseId2); + expect(integrity2.data.hasIssues).toEqual(true); + expect(integrity2.data.linkFieldIssues.length).toEqual(1); + + await fixBaseIntegrity(baseId2); + + const integrity3 = await checkBaseIntegrity(baseId2); + expect(integrity3.data.hasIssues).toEqual(false); + + // test single link + await executeKnex( + db(base2table1.dbTableName) + .where('__id', base2table1.records[0].id) + .update({ + [linkField.dbFieldName]: db.raw(`jsonb_set( + "${linkField.dbFieldName}", + '{0,id}', + '"xxx"' + )`), + }) + ); + + const record2 = await getRecord(base2table1.id, base2table1.records[0].id); + expect(record2.data.fields[linkField.name]).toEqual([{ id: 'xxx', title: 'b1' }]); + + const integrity4 = await checkBaseIntegrity(baseId2); + expect(integrity4.data.hasIssues).toEqual(true); + + await fixBaseIntegrity(baseId2); + + const integrity5 = await checkBaseIntegrity(baseId2); + expect(integrity5.data.hasIssues).toEqual(false); + }); }); }); diff --git a/apps/nextjs-app/.env.development b/apps/nextjs-app/.env.development index 27e77e4f41..c123424afd 100644 --- a/apps/nextjs-app/.env.development +++ b/apps/nextjs-app/.env.development @@ -15,10 +15,8 @@ LOG_LEVEL=info PORT=3000 SOCKET_PORT=3001 -PUBLIC_ORIGIN=http://127.0.0.1:3000 +PUBLIC_ORIGIN=http://localhost:3000 -# storage service url prefix -STORAGE_PREFIX=http://localhost:3000 # DATABASE_URL # @see https://www.prisma.io/docs/reference/database-reference/connection-urls#examples PRISMA_DATABASE_URL=postgresql://teable:teable@127.0.0.1:5432/teable?schema=public&statement_cache_size=1 diff --git a/package.json b/package.json index 28f2f1f4bd..bbf9682ce6 100644 --- a/package.json +++ b/package.json @@ -78,7 +78,7 @@ }, "engines": { "node": ">=20.0.0", - "pnpm": ">=8.15.0", + "pnpm": ">=9.13.0", "npm": "please-use-pnpm" }, "packageManager": "pnpm@9.13.0" diff --git a/packages/openapi/src/integrity/link-check.ts b/packages/openapi/src/integrity/link-check.ts index fa216e8c6a..c1e43f8853 100644 --- a/packages/openapi/src/integrity/link-check.ts +++ b/packages/openapi/src/integrity/link-check.ts @@ -11,7 +11,8 @@ export enum IntegrityIssueType { ForeignKeyNotFound = 'ForeignKeyNotFound', SelfKeyNotFound = 'SelfKeyNotFound', SymmetricFieldNotFound = 'SymmetricFieldNotFound', - InvalidRecordReference = 'InvalidRecordReference', + MissingRecordReference = 'MissingRecordReference', + InvalidLinkReference = 'InvalidLinkReference', ForeignKeyHostTableNotFound = 'ForeignKeyHostTableNotFound', } diff --git a/plugins/README.md b/plugins/README.md index f39a3a0c38..38137390d3 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -1 +1,229 @@ -# Plugins \ No newline at end of file +# Plugin Development Guide + +## Table of Contents +1. [Overview](#overview) +2. [Plugin Structure](#plugin-structure) +3. [Creating a Plugin](#creating-a-plugin) +4. [Plugin Configuration](#plugin-configuration) +5. [Plugin Bridge](#plugin-bridge) +6. [Internationalization](#internationalization) +7. [Publishing](#publishing) + +## Overview + +Plugins are extensions that can be integrated into different positions within the application. All official plugins are managed within a single Next.js project (`plugins`), allowing for code sharing and simplified maintenance. + +### Supported Positions + +Plugins can be integrated into two main positions: + +````typescript:packages/openapi/src/plugin/types.ts +export enum PluginPosition { + Dashboard = 'dashboard', + View = 'view', +} +```` + +## Plugin Structure + +### Project Structure +The plugins project uses Next.js App Router structure: + +````bash +plugins/ +├── src/ +│ ├── app/ +│ │ ├── chart/ # Chart plugin +│ │ │ ├── components/ +│ │ │ ├── page.tsx +│ │ │ └── favicon.ico +│ │ ├── sheet-form/ # Sheet Form plugin +│ │ │ ├── components/ +│ │ │ ├── page.tsx +│ │ │ └── favicon.ico +│ ├── components/ # Shared components +│ ├── locales/ # i18n translations +│ │ ├── chart/ +│ │ │ ├── en.json +│ │ │ └── zh.json +│ │ └── sheet-form/ +│ │ ├── en.json +│ │ └── zh.json +│ └── types.ts +├── package.json +└── tsconfig.json +```` + +### Plugin Page Structure +Each plugin should have its own directory under `src/app/` with the following structure: + +````typescript:plugins/src/app/chart/page.tsx +import type { Metadata } from 'next'; +import { EnvProvider } from '../../components/EnvProvider'; +import { I18nProvider } from '../../components/I18nProvider'; +import QueryClientProvider from '../../components/QueryClientProvider'; +import { PageType } from '../../components/types'; +import enCommonJson from '../../locales/chart/en.json'; +import zhCommonJson from '../../locales/chart/zh.json'; +import { Pages } from './components/Pages'; + +export async function generateMetadata({ searchParams }: Props): Promise { + const lang = searchParams.lang; + return { + title: lang === 'zh' ? '图表' : 'Chart', + icons: icon.src, + }; +} + +export default async function Home(props: { searchParams: IPageParams }) { + return ( +
+ + + + + + + +
+ ); +} +```` + +## Creating a Plugin + +### 1. Create Plugin Directory +Add a new directory under `src/app/` for your plugin: + +````bash +src/app/my-plugin/ +├── components/ +├── page.tsx +└── favicon.ico +```` + +### 2. Configure Plugin +Create a plugin configuration file: + +````typescript:apps/nestjs-backend/src/features/plugin/official/config/my-plugin.ts +import { PluginPosition } from '@teable/openapi'; +import type { IOfficialPluginConfig } from './types'; + +export const myPluginConfig: IOfficialPluginConfig = { + id: 'plg-my-plugin', + name: 'My Plugin', + description: 'Plugin description', + detailDesc: `Detailed description with markdown support`, + helpUrl: 'https://teable.io', + positions: [PluginPosition.Dashboard], + i18n: { + zh: { + name: '我的插件', + helpUrl: 'https://teable.cn', + description: '插件描述', + detailDesc: '详细描述', + }, + }, + logoPath: 'static/plugin/my-plugin.png', + pluginUserId: 'plgmypluginuser', + avatarPath: 'static/plugin/my-plugin.png', +}; +```` + +## Plugin Bridge + +The Plugin Bridge enables communication between your plugin and the main application. + +### Bridge Methods + +````typescript:apps/nextjs-app/src/features/app/components/plugin/PluginRender.tsx +const methods: IParentBridgeMethods = { + expandRecord: (recordIds) => { + console.log('expandRecord', recordIds); + }, + updateStorage: (storage) => { + return updateDashboardPluginStorage(baseId, positionId, pluginInstallId, storage).then( + (res) => res.data.storage ?? {} + ); + }, + getAuthCode: () => { + return pluginGetAuthCode(pluginId, baseId).then((res) => res.data); + }, + expandPlugin: () => { + onExpand?.(); + }, +}; +```` + +### Initializing the Bridge + +````typescript:packages/sdk/src/plugin-bridge/bridge.ts +export const initializeBridge = async () => { + if (typeof window === 'undefined') { + return; + } + const pluginBridge = new PluginBridge(); + const bridge = await pluginBridge.init(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (window as any)._teable_plugin_bridge = bridge; + return bridge; +}; +```` + +## Internationalization + +Add translations for your plugin under `src/locales/[plugin-name]/`: + +````json:plugins/src/locales/my-plugin/zh.json +{ + "title": "我的插件", + "description": "插件描述", + "actions": { + "save": "保存", + "cancel": "取消" + } +} +```` + +## Publishing + +1. **Development Status**: Plugins start in `developing` status +2. **Review Process**: Submit for review using the plugin management interface +3. **Publication**: Once approved, the plugin will be published and available in the plugin center + +### Plugin Status Flow + +````typescript:packages/openapi/src/plugin/types.ts +export enum PluginStatus { + Developing = 'developing', + Reviewing = 'reviewing', + Published = 'published', +} +```` + +## Best Practices + +1. **Code Sharing**: Utilize shared components and utilities from the plugins project +2. **Consistent UI**: Follow the design patterns used by other plugins +3. **Error Handling**: Implement proper error handling and display user-friendly messages +4. **Responsive Design**: Ensure your plugin works well in different container sizes +5. **Performance**: Optimize loading time and resource usage +6. **Security**: Never expose sensitive information in the client-side code +7. **Documentation**: Document your plugin's features and configuration options + +For more detailed information and API references, please refer to our complete API documentation. + +## Best Practices + +1. **Error Handling**: Implement proper error handling and display user-friendly messages +2. **Responsive Design**: Ensure your plugin works well in different container sizes +3. **Performance**: Optimize loading time and resource usage +4. **Security**: Never expose sensitive information in the client-side code +5. **Documentation**: Provide clear documentation for your plugin's features and configuration options + +For more detailed information and API references, please refer to our complete API documentation.