feat: add integrity check

Author: tea-artist

apps/nestjs-backend/src/app.module.ts

@@ -18,6 +18,7 @@ import { ExportOpenApiModule } from './features/export/open-api/export-open-api.
 import { FieldOpenApiModule } from './features/field/open-api/field-open-api.module';
 import { HealthModule } from './features/health/health.module';
 import { ImportOpenApiModule } from './features/import/open-api/import-open-api.module';
+import { IntegrityModule } from './features/integrity/integrity.module';
 import { InvitationModule } from './features/invitation/invitation.module';
 import { NextModule } from './features/next/next.module';
 import { NotificationModule } from './features/notification/notification.module';
@@ -44,6 +45,7 @@ export const appModules = {
     NextModule,
     FieldOpenApiModule,
     BaseModule,
+    IntegrityModule,
     ChatModule,
     AttachmentsModule,
     WsModule,

apps/nestjs-backend/src/db-provider/db.provider.interface.ts

@@ -1,4 +1,4 @@
-import type { DriverClient, IFilter, ILookupOptionsVo, ISortItem } from '@teable/core';
+import type { DriverClient, FieldType, IFilter, ILookupOptionsVo, ISortItem } from '@teable/core';
 import type { Prisma } from '@teable/db-main-prisma';
 import type { IAggregationField, ISearchIndexByQueryRo } from '@teable/openapi';
 import type { Knex } from 'knex';
@@ -72,6 +72,8 @@ export interface IDbProvider {
     prisma: Prisma.TransactionClient
   ): Promise<boolean>;
 
+  checkTableExist(tableName: string): string;
+
   dropColumnAndIndex(tableName: string, columnName: string, indexName: string): string[];
 
   modifyColumnSchema(tableName: string, columnName: string, schemaType: SchemaType): string[];
@@ -165,5 +167,5 @@ export interface IDbProvider {
 
   lookupOptionsQuery(optionsKey: keyof ILookupOptionsVo, value: string): string;
 
-  optionsQuery(optionsKey: string, value: string): string;
+  optionsQuery(type: FieldType, optionsKey: string, value: string): string;
 }

apps/nestjs-backend/src/db-provider/postgres.provider.ts

@@ -1,6 +1,6 @@
 /* eslint-disable sonarjs/no-duplicate-string */
 import { Logger } from '@nestjs/common';
-import type { IFilter, ILookupOptionsVo, ISortItem } from '@teable/core';
+import type { FieldType, IFilter, ILookupOptionsVo, ISortItem } from '@teable/core';
 import { DriverClient } from '@teable/core';
 import type { PrismaClient } from '@teable/db-main-prisma';
 import type { IAggregationField, ISearchIndexByQueryRo } from '@teable/openapi';
@@ -75,6 +75,16 @@ export class PostgresProvider implements IDbProvider {
     return res[0].exists;
   }
 
+  checkTableExist(tableName: string): string {
+    const [schemaName, dbTableName] = this.splitTableName(tableName);
+    return this.knex
+      .raw(
+        'SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_schema = ? AND table_name = ?) AS exists',
+        [schemaName, dbTableName]
+      )
+      .toQuery();
+  }
+
   renameColumn(tableName: string, oldName: string, newName: string): string[] {
     return this.knex.schema
       .alterTable(tableName, (table) => {
@@ -424,20 +434,29 @@ export class PostgresProvider implements IDbProvider {
   lookupOptionsQuery(optionsKey: keyof ILookupOptionsVo, value: string): string {
     return this.knex('field')
       .select({
+        tableId: 'table_id',
         id: 'id',
+        type: 'type',
+        name: 'name',
         lookupOptions: 'lookup_options',
       })
+      .whereNull('deleted_time')
       .whereRaw(`lookup_options::json->>'${optionsKey}' = ?`, [value])
       .toQuery();
   }
 
-  optionsQuery(optionsKey: string, value: string): string {
+  optionsQuery(type: FieldType, optionsKey: string, value: string): string {
     return this.knex('field')
       .select({
+        tableId: 'table_id',
         id: 'id',
+        type: 'type',
+        name: 'name',
         options: 'options',
       })
+      .whereNull('deleted_time')
       .whereRaw(`options::json->>'${optionsKey}' = ?`, [value])
+      .where('type', type)
       .toQuery();
   }
 }

apps/nestjs-backend/src/db-provider/sqlite.provider.ts

@@ -1,6 +1,6 @@
 /* eslint-disable sonarjs/no-duplicate-string */
 import { Logger } from '@nestjs/common';
-import type { IFilter, ILookupOptionsVo, ISortItem } from '@teable/core';
+import type { FieldType, IFilter, ILookupOptionsVo, ISortItem } from '@teable/core';
 import { DriverClient } from '@teable/core';
 import type { PrismaClient } from '@teable/db-main-prisma';
 import type { IAggregationField, ISearchIndexByQueryRo } from '@teable/openapi';
@@ -65,6 +65,18 @@ export class SqliteProvider implements IDbProvider {
     return columns.some((column) => column.name === columnName);
   }
 
+  checkTableExist(tableName: string): string {
+    return this.knex
+      .raw(
+        `SELECT EXISTS (
+          SELECT 1 FROM sqlite_master 
+          WHERE type='table' AND name = ?
+        ) as exists`,
+        [tableName]
+      )
+      .toQuery();
+  }
+
   renameColumn(tableName: string, oldName: string, newName: string): string[] {
     return [
       this.knex
@@ -382,18 +394,25 @@ export class SqliteProvider implements IDbProvider {
     return this.knex('field')
       .select({
         id: 'id',
+        type: 'type',
+        name: 'name',
         lookupOptions: 'lookup_options',
       })
+      .whereNull('deleted_time')
       .whereRaw(`json_extract(lookup_options, '$."${optionsKey}"') = ?`, [value])
       .toQuery();
   }
 
-  optionsQuery(optionsKey: string, value: string): string {
+  optionsQuery(type: FieldType, optionsKey: string, value: string): string {
     return this.knex('field')
       .select({
         id: 'id',
+        type: 'type',
+        name: 'name',
         options: 'options',
       })
+      .where('type', type)
+      .whereNull('deleted_time')
       .whereRaw(`json_extract(options, '$."${optionsKey}"') = ?`, [value])
       .toQuery();
   }

apps/nestjs-backend/src/features/calculation/reference.service.ts

@@ -343,6 +343,10 @@ export class ReferenceService {
       if (!fromRecordIds?.length && !toRecordIds?.length) {
         continue;
       }
+
+      console.log('order', JSON.stringify(order, null, 2));
+      console.log('order:fromRecordIds', fromRecordIds);
+      console.log('order:toRecordIds', toRecordIds);
       const relatedRecordItems = await this.getAffectedRecordItems({
         fieldId,
         fieldMap,
@@ -351,6 +355,7 @@ export class ReferenceService {
         fkRecordMap,
         tableId2DbTableName,
       });
+      console.log('relatedRecordItems', relatedRecordItems);
 
       if (field.lookupOptions || field.type === FieldType.Link) {
         await this.calculateLinkRelatedRecords({
@@ -583,6 +588,13 @@ export class ReferenceService {
       : (field.options as ILinkFieldOptions);
     const { relationship } = lookupOptions;
     const linkFieldId = field.lookupOptions ? field.lookupOptions.linkFieldId : field.id;
+
+    if (!recordItem.record?.fields) {
+      console.log('recordItem', JSON.stringify(recordItem, null, 2));
+      console.log('recordItem.field', field);
+      throw new InternalServerErrorException('record fields is undefined');
+    }
+
     const cellValue = recordItem.record.fields[linkFieldId];
     const dependenciesIndexed = keyBy(dependencies, 'id');
 

apps/nestjs-backend/src/features/integrity/integrity.controller.ts

@@ -0,0 +1,23 @@
+import { Controller, Get, Param, Post, UseGuards } from '@nestjs/common';
+import type { IIntegrityCheckVo } from '@teable/openapi';
+import { Permissions } from '../auth/decorators/permissions.decorator';
+import { PermissionGuard } from '../auth/guard/permission.guard';
+import { LinkIntegrityService } from './link-integrity.service';
+
+@UseGuards(PermissionGuard)
+@Controller('api/integrity')
+export class IntegrityController {
+  constructor(private readonly linkIntegrityService: LinkIntegrityService) {}
+
+  @Permissions('table|create')
+  @Get('base/:baseId/link-check')
+  async checkBaseIntegrity(@Param('baseId') baseId: string): Promise<IIntegrityCheckVo> {
+    return await this.linkIntegrityService.linkIntegrityCheck(baseId);
+  }
+
+  @Permissions('table|create')
+  @Post('base/:baseId/link-fix')
+  async fixBaseIntegrity(@Param('baseId') baseId: string): Promise<void> {
+    return await this.linkIntegrityService.linkIntegrityFix(baseId);
+  }
+}

apps/nestjs-backend/src/features/integrity/integrity.module.ts

@@ -0,0 +1,10 @@
+import { Module } from '@nestjs/common';
+import { IntegrityController } from './integrity.controller';
+import { LinkIntegrityService } from './link-integrity.service';
+
+@Module({
+  controllers: [IntegrityController],
+  providers: [LinkIntegrityService],
+  exports: [LinkIntegrityService],
+})
+export class IntegrityModule {}