feat: link field integrity check

README.md

@@ -204,6 +204,14 @@ cd apps/nestjs-backend
 pnpm dev
 ```
 
+By default, the plugin development server is not started. To preview and develop plugins, run:
+```sh
+cd plugins
+pnpm dev
+```
+This will start the plugin development server on port 3002.
+
+
 ## Why Teable?
 
 No-code tools have significantly speed up how we get things done, allowing non-tech users to build amazing apps and changing the way many work and live. People like using spreadsheet-like UI to handle their data because it's easy, flexible, and great for team collaboration. They also prefer designing their app screens without being stuck with clunky templates.

apps/nestjs-backend/src/db-provider/postgres.provider.ts

@@ -450,9 +450,22 @@ export class PostgresProvider implements IDbProvider {
       .select({
         tableId: 'table_id',
         id: 'id',
-        type: 'type',
         name: 'name',
+        description: 'description',
+        notNull: 'not_null',
+        unique: 'unique',
+        isPrimary: 'is_primary',
+        dbFieldName: 'db_field_name',
+        isComputed: 'is_computed',
+        isPending: 'is_pending',
+        hasError: 'has_error',
+        dbFieldType: 'db_field_type',
+        isMultipleCellValue: 'is_multiple_cell_value',
+        isLookup: 'is_lookup',
+        lookupOptions: 'lookup_options',
+        type: 'type',
         options: 'options',
+        cellValueType: 'cell_value_type',
       })
       .whereNull('deleted_time')
       .whereNull('is_lookup')

apps/nestjs-backend/src/db-provider/sqlite.provider.ts

@@ -406,10 +406,24 @@ export class SqliteProvider implements IDbProvider {
   optionsQuery(type: FieldType, optionsKey: string, value: string): string {
     return this.knex('field')
       .select({
+        tableId: 'table_id',
         id: 'id',
-        type: 'type',
         name: 'name',
+        description: 'description',
+        notNull: 'not_null',
+        unique: 'unique',
+        isPrimary: 'is_primary',
+        dbFieldName: 'db_field_name',
+        isComputed: 'is_computed',
+        isPending: 'is_pending',
+        hasError: 'has_error',
+        dbFieldType: 'db_field_type',
+        isMultipleCellValue: 'is_multiple_cell_value',
+        isLookup: 'is_lookup',
+        lookupOptions: 'lookup_options',
+        type: 'type',
         options: 'options',
+        cellValueType: 'cell_value_type',
       })
       .where('type', type)
       .whereNull('is_lookup')

apps/nestjs-backend/src/features/calculation/batch.service.ts

@@ -1,5 +1,5 @@
 /* eslint-disable @typescript-eslint/naming-convention */
-import { Injectable, Logger } from '@nestjs/common';
+import { BadRequestException, Injectable, Logger } from '@nestjs/common';
 import type { IOtOperation } from '@teable/core';
 import { IdPrefix, RecordOpBuilder } from '@teable/core';
 import { PrismaService } from '@teable/db-main-prisma';
@@ -104,6 +104,12 @@ export class BatchService {
     );
     const versionGroup = keyBy(raw, '__id');
 
+    opsPair.map(([recordId]) => {
+      if (!versionGroup[recordId]) {
+        throw new BadRequestException(`Record ${recordId} not found in ${tableId}`);
+      }
+    });
+
     const opsData = this.buildRecordOpsData(opsPair, versionGroup);
     if (!opsData.length) return;
 

apps/nestjs-backend/src/features/calculation/reference.service.ts

@@ -617,7 +617,7 @@ export class ReferenceService {
             const result = dependenciesIndexed[v.id];
             if (!result) {
               throw new InternalServerErrorException(
-                `Record not found for: ${JSON.stringify(v)}, fieldId: ${field.id}`
+                `Record not found for: ${JSON.stringify(v)}, fieldId: ${field.id}, when calculate ${JSON.stringify(recordItem.record.id)}`
               );
             }
             return result;

apps/nestjs-backend/src/features/integrity/foreign-key.service.ts

@@ -0,0 +1,175 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { FieldType, type ILinkFieldOptions } from '@teable/core';
+import { Prisma, PrismaService } from '@teable/db-main-prisma';
+import { IntegrityIssueType, type IIntegrityIssue } from '@teable/openapi';
+import { Knex } from 'knex';
+import { InjectModel } from 'nest-knexjs';
+import type { LinkFieldDto } from '../field/model/field-dto/link-field.dto';
+
+@Injectable()
+export class ForeignKeyIntegrityService {
+  private readonly logger = new Logger(ForeignKeyIntegrityService.name);
+
+  constructor(
+    private readonly prismaService: PrismaService,
+    @InjectModel('CUSTOM_KNEX') private readonly knex: Knex
+  ) {}
+
+  async getIssues(tableId: string, field: LinkFieldDto): Promise<IIntegrityIssue[]> {
+    const { foreignTableId, fkHostTableName, foreignKeyName, selfKeyName } = field.options;
+    const issues: IIntegrityIssue[] = [];
+
+    const { name: selfTableName, dbTableName: selfTableDbTableName } =
+      await this.prismaService.tableMeta.findFirstOrThrow({
+        where: { id: tableId, deletedTime: null },
+        select: { name: true, dbTableName: true },
+      });
+
+    const { name: foreignTableName, dbTableName: foreignTableDbTableName } =
+      await this.prismaService.tableMeta.findFirstOrThrow({
+        where: { id: foreignTableId, deletedTime: null },
+        select: { name: true, dbTableName: true },
+      });
+
+    // Check self references
+    if (selfTableDbTableName !== fkHostTableName) {
+      const selfIssues = await this.checkInvalidReferences({
+        fkHostTableName,
+        targetTableName: selfTableDbTableName,
+        keyName: selfKeyName,
+        field,
+        referencedTableName: selfTableName,
+        isSelfReference: true,
+      });
+      issues.push(...selfIssues);
+    }
+
+    // Check foreign references
+    if (foreignTableDbTableName !== fkHostTableName) {
+      const foreignIssues = await this.checkInvalidReferences({
+        fkHostTableName,
+        targetTableName: foreignTableDbTableName,
+        keyName: foreignKeyName,
+        field,
+        referencedTableName: foreignTableName,
+        isSelfReference: false,
+      });
+      issues.push(...foreignIssues);
+    }
+
+    return issues;
+  }
+
+  private async checkInvalidReferences({
+    fkHostTableName,
+    targetTableName,
+    keyName,
+    field,
+    referencedTableName,
+    isSelfReference,
+  }: {
+    fkHostTableName: string;
+    targetTableName: string;
+    keyName: string;
+    field: { id: string; name: string };
+    referencedTableName: string;
+    isSelfReference: boolean;
+  }): Promise<IIntegrityIssue[]> {
+    const issues: IIntegrityIssue[] = [];
+
+    const invalidQuery = this.knex(fkHostTableName)
+      .leftJoin(targetTableName, `${fkHostTableName}.${keyName}`, `${targetTableName}.__id`)
+      .whereNull(`${targetTableName}.__id`)
+      .count(`${fkHostTableName}.${keyName} as count`)
+      .first()
+      .toQuery();
+
+    try {
+      const invalidRefs =
+        await this.prismaService.$queryRawUnsafe<{ count: bigint }[]>(invalidQuery);
+      const refCount = Number(invalidRefs[0]?.count || 0);
+
+      if (refCount > 0) {
+        const message = isSelfReference
+          ? `Found ${refCount} invalid self references in table ${referencedTableName}`
+          : `Found ${refCount} invalid foreign references to table ${referencedTableName}`;
+
+        issues.push({
+          type: IntegrityIssueType.MissingRecordReference,
+          message: `${message} (Field Name: ${field.name}, Field ID: ${field.id})`,
+        });
+      }
+    } catch (error) {
+      if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === 'P2010') {
+        console.error('error ignored:', error);
+      } else {
+        throw error;
+      }
+    }
+
+    return issues;
+  }
+
+  async fix(_tableId: string, fieldId: string): Promise<IIntegrityIssue | undefined> {
+    const field = await this.prismaService.field.findFirstOrThrow({
+      where: { id: fieldId, type: FieldType.Link, isLookup: null, deletedTime: null },
+    });
+
+    const options = JSON.parse(field.options as string) as ILinkFieldOptions;
+    const { foreignTableId, fkHostTableName, foreignKeyName, selfKeyName } = options;
+    const foreignTable = await this.prismaService.tableMeta.findFirstOrThrow({
+      where: { id: foreignTableId, deletedTime: null },
+      select: { id: true, name: true, dbTableName: true },
+    });
+
+    let totalFixed = 0;
+
+    // Fix invalid self references
+    if (fkHostTableName !== fkHostTableName) {
+      const selfDeleted = await this.deleteMissingReferences({
+        fkHostTableName,
+        targetTableName: fkHostTableName,
+        keyName: selfKeyName,
+      });
+      totalFixed += selfDeleted;
+    }
+
+    // Fix invalid foreign references
+    if (foreignTable.dbTableName !== fkHostTableName) {
+      const foreignDeleted = await this.deleteMissingReferences({
+        fkHostTableName,
+        targetTableName: foreignTable.dbTableName,
+        keyName: foreignKeyName,
+      });
+      totalFixed += foreignDeleted;
+    }
+
+    if (totalFixed > 0) {
+      return {
+        type: IntegrityIssueType.MissingRecordReference,
+        message: `Fixed ${totalFixed} invalid references and inconsistent links for link field (Field Name: ${field.name}, Field ID: ${field.id})`,
+      };
+    }
+  }
+
+  private async deleteMissingReferences({
+    fkHostTableName,
+    targetTableName,
+    keyName,
+  }: {
+    fkHostTableName: string;
+    targetTableName: string;
+    keyName: string;
+  }) {
+    const deleteQuery = this.knex(fkHostTableName)
+      .whereNotExists(
+        this.knex
+          .select('__id')
+          .from(targetTableName)
+          .where('__id', this.knex.ref(`${fkHostTableName}.${keyName}`))
+      )
+      .delete()
+      .toQuery();
+    return await this.prismaService.$executeRawUnsafe(deleteQuery);
+  }
+}

apps/nestjs-backend/src/features/integrity/integrity.module.ts

@@ -1,10 +1,12 @@
 import { Module } from '@nestjs/common';
+import { ForeignKeyIntegrityService } from './foreign-key.service';
 import { IntegrityController } from './integrity.controller';
+import { LinkFieldIntegrityService } from './link-field.service';
 import { LinkIntegrityService } from './link-integrity.service';
 
 @Module({
   controllers: [IntegrityController],
-  providers: [LinkIntegrityService],
+  providers: [ForeignKeyIntegrityService, LinkFieldIntegrityService, LinkIntegrityService],
   exports: [LinkIntegrityService],
 })
 export class IntegrityModule {}