Skip to content

Commit 706b1c8

Browse files
boris-wboris-w
authored
refactor: cut thumbnail not block record operations (#1024)
* refactor: cut thumbnail not block record operations * fix: add event and fix e2e * fix: e2e * fix: local unit tests
1 parent c398cf4 commit 706b1c8

File tree

16 files changed

+136
-99
lines changed

16 files changed

+136
-99
lines changed

‎apps/nestjs-backend/src/event-emitter/events/event.enum.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,4 +61,6 @@ export enum Events {
6161

6262
WORKFLOW_ACTIVATE = 'workflow.activate',
6363
WORKFLOW_DEACTIVATE = 'workflow.deactivate',
64+
65+
CROP_IMAGE = 'crop.image',
6466
}

‎apps/nestjs-backend/src/features/attachments/attachments-storage.service.ts

Lines changed: 51 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,10 @@ import { PrismaService } from '@teable/db-main-prisma';
33
import { UploadType } from '@teable/openapi';
44
import { CacheService } from '../../cache/cache.service';
55
import { IStorageConfig, StorageConfig } from '../../configs/storage';
6+
import { EventEmitterService } from '../../event-emitter/event-emitter.service';
7+
import { Events } from '../../event-emitter/events';
68
import {
9+
generateTableThumbnailPath,
710
getTableThumbnailSize,
811
getTableThumbnailToken,
912
} from '../../utils/generate-table-thumbnail-path';
@@ -17,6 +20,7 @@ export class AttachmentsStorageService {
1720
constructor(
1821
private readonly cacheService: CacheService,
1922
private readonly prismaService: PrismaService,
23+
private readonly eventEmitterService: EventEmitterService,
2024
@StorageConfig() private readonly storageConfig: IStorageConfig,
2125
@InjectStorageAdapter() private readonly storageAdapter: StorageAdapter
2226
) {}
@@ -71,7 +75,9 @@ export class AttachmentsStorageService {
7175
let url = previewCache?.url;
7276
if (!url) {
7377
url = await this.storageAdapter.getPreviewUrl(bucket, path, expiresIn, respHeaders);
74-
78+
if (!url) {
79+
throw new BadRequestException(`Invalid token: ${token}`);
80+
}
7581
await this.cacheService.set(
7682
`attachment:preview:${token}`,
7783
{
@@ -84,38 +90,63 @@ export class AttachmentsStorageService {
8490
return url;
8591
}
8692

87-
async getTableAttachmentThumbnailUrl(smThumbnailPath?: string, lgThumbnailPath?: string) {
88-
const smThumbnailUrl = smThumbnailPath
89-
? await this.getPreviewUrlByPath(
90-
StorageAdapter.getBucket(UploadType.Table),
91-
smThumbnailPath,
92-
getTableThumbnailToken(smThumbnailPath)
93-
)
94-
: undefined;
95-
const lgThumbnailUrl = lgThumbnailPath
96-
? await this.getPreviewUrlByPath(
97-
StorageAdapter.getBucket(UploadType.Table),
98-
lgThumbnailPath,
99-
getTableThumbnailToken(lgThumbnailPath)
100-
)
101-
: undefined;
93+
private async getTableThumbnailUrl(path: string, token: string) {
94+
const previewCache = await this.cacheService.get(`attachment:preview:${token}`);
95+
if (previewCache?.url) {
96+
return previewCache.url;
97+
}
98+
const url = await this.storageAdapter.getPreviewUrl(
99+
StorageAdapter.getBucket(UploadType.Table),
100+
path,
101+
second(this.storageConfig.urlExpireIn)
102+
);
103+
if (url) {
104+
await this.cacheService.set(
105+
`attachment:preview:${token}`,
106+
{
107+
url,
108+
expiresIn: second(this.storageConfig.urlExpireIn),
109+
},
110+
second(this.storageConfig.urlExpireIn)
111+
);
112+
}
113+
return url;
114+
}
115+
116+
async getTableAttachmentThumbnailUrl(path: string) {
117+
const { smThumbnailPath, lgThumbnailPath } = generateTableThumbnailPath(path);
118+
const smThumbnailUrl = await this.getTableThumbnailUrl(
119+
smThumbnailPath,
120+
getTableThumbnailToken(smThumbnailPath)
121+
);
122+
const lgThumbnailUrl = await this.getTableThumbnailUrl(
123+
lgThumbnailPath,
124+
getTableThumbnailToken(lgThumbnailPath)
125+
);
102126
return { smThumbnailUrl, lgThumbnailUrl };
103127
}
104128

105129
async cutTableImage(bucket: string, path: string, width: number, height: number) {
106130
const { smThumbnail, lgThumbnail } = getTableThumbnailSize(width, height);
107-
const cutSmThumbnailPath = await this.storageAdapter.cutImage(
131+
const { smThumbnailPath, lgThumbnailPath } = generateTableThumbnailPath(path);
132+
const cutSmThumbnailPath = await this.storageAdapter.cropImage(
108133
bucket,
109134
path,
110135
smThumbnail.width,
111-
smThumbnail.height
136+
smThumbnail.height,
137+
smThumbnailPath
112138
);
113-
const cutLgThumbnailPath = await this.storageAdapter.cutImage(
139+
const cutLgThumbnailPath = await this.storageAdapter.cropImage(
114140
bucket,
115141
path,
116142
lgThumbnail.width,
117-
lgThumbnail.height
143+
lgThumbnail.height,
144+
lgThumbnailPath
118145
);
146+
this.eventEmitterService.emit(Events.CROP_IMAGE, {
147+
bucket,
148+
path,
149+
});
119150
return {
120151
smThumbnailPath: cutSmThumbnailPath,
121152
lgThumbnailPath: cutLgThumbnailPath,

‎apps/nestjs-backend/src/features/attachments/plugins/adapter.ts

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ export default abstract class StorageAdapter {
8383
// eslint-disable-next-line @typescript-eslint/no-explicit-any
8484
[key: string]: any;
8585
}
86-
): Promise<string>;
86+
): Promise<string | undefined>;
8787

8888
/**
8989
* uploadFile with file path
@@ -119,7 +119,14 @@ export default abstract class StorageAdapter {
119119
* @param path path name
120120
* @param width width
121121
* @param height height
122+
* @param newPath save as new path
122123
* @returns cut image url
123124
*/
124-
abstract cutImage(bucket: string, path: string, width: number, height: number): Promise<string>;
125+
abstract cropImage(
126+
bucket: string,
127+
path: string,
128+
width: number,
129+
height: number,
130+
newPath?: string
131+
): Promise<string>;
125132
}

‎apps/nestjs-backend/src/features/attachments/plugins/local.spec.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -316,15 +316,15 @@ describe('LocalStorage', () => {
316316
});
317317
});
318318

319-
describe('getPreviewUrl', () => {
319+
describe('getPreviewUrlInner', () => {
320320
it('should get preview URL', async () => {
321321
const mockBucket = 'mock-bucket';
322322
const mockPath = 'mock/file/path';
323323
const mockExpiresIn = 3600;
324324

325325
vi.spyOn(storage.expireTokenEncryptor, 'encrypt').mockReturnValueOnce('mock-token');
326326

327-
const result = await storage.getPreviewUrl(
327+
const result = await storage.getPreviewUrlInner(
328328
mockBucket,
329329
mockPath,
330330
mockExpiresIn,

‎apps/nestjs-backend/src/features/attachments/plugins/local.ts

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ import { Encryptor } from '../../../utils/encryptor';
1515
import { second } from '../../../utils/second';
1616
import StorageAdapter from './adapter';
1717
import type { ILocalFileUpload, IObjectMeta, IPresignParams, IRespHeaders } from './types';
18-
import { generateCutImagePath } from './utils';
1918

2019
interface ITokenEncryptor {
2120
expiresDate: number;
@@ -212,14 +211,25 @@ export class LocalStorage implements StorageAdapter {
212211
path: string,
213212
expiresIn: number = second(this.config.urlExpireIn),
214213
respHeaders?: IRespHeaders
215-
): Promise<string> {
214+
): Promise<string | undefined> {
215+
if (!fse.existsSync(resolve(this.storageDir, bucket, path))) {
216+
return undefined;
217+
}
218+
return this.getPreviewUrlInner(bucket, path, expiresIn, respHeaders);
219+
}
220+
221+
async getPreviewUrlInner(
222+
bucket: string,
223+
path: string,
224+
expiresIn: number,
225+
respHeaders?: IRespHeaders
226+
) {
216227
const url = this.getUrl(bucket, path, {
217228
expiresDate: Math.floor(Date.now() / 1000) + expiresIn,
218229
respHeaders,
219230
});
220231
return this.baseConfig.storagePrefix + join('/', url);
221232
}
222-
223233
verifyReadToken(token: string) {
224234
try {
225235
const { expiresDate, respHeaders } = this.expireTokenEncryptor.decrypt(token);
@@ -281,14 +291,15 @@ export class LocalStorage implements StorageAdapter {
281291
};
282292
}
283293

284-
async cutImage(bucket: string, path: string, width: number, height: number) {
285-
const newPath = generateCutImagePath(path, width, height);
294+
async cropImage(bucket: string, path: string, width: number, height: number, _newPath?: string) {
295+
const newPath = _newPath || `${path}_${width}_${height}`;
286296
const resizedImagePath = resolve(this.storageDir, bucket, newPath);
287297
if (fse.existsSync(resizedImagePath)) {
288298
return newPath;
289299
}
300+
290301
const imagePath = resolve(this.storageDir, bucket, path);
291-
const image = sharp(imagePath);
302+
const image = sharp(imagePath, { failOn: 'none', unlimited: true });
292303
const metadata = await image.metadata();
293304
if (!metadata.width || !metadata.height) {
294305
throw new BadRequestException('Invalid image');

‎apps/nestjs-backend/src/features/attachments/plugins/minio.ts

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ import { IStorageConfig, StorageConfig } from '../../../configs/storage';
1010
import { second } from '../../../utils/second';
1111
import StorageAdapter from './adapter';
1212
import type { IPresignParams, IPresignRes, IRespHeaders } from './types';
13-
import { generateCutImagePath } from './utils';
1413

1514
@Injectable()
1615
export class MinioStorage implements StorageAdapter {
@@ -125,6 +124,9 @@ export class MinioStorage implements StorageAdapter {
125124
expiresIn: number = second(this.config.urlExpireIn),
126125
respHeaders?: IRespHeaders
127126
) {
127+
if (!(await this.fileExists(bucket, path))) {
128+
return;
129+
}
128130
const { 'Content-Disposition': contentDisposition, ...headers } = respHeaders ?? {};
129131
return this.minioClient.presignedGetObject(bucket, path, expiresIn, {
130132
...headers,
@@ -178,8 +180,8 @@ export class MinioStorage implements StorageAdapter {
178180
}
179181
}
180182

181-
async cutImage(bucket: string, path: string, width: number, height: number) {
182-
const newPath = generateCutImagePath(path, width, height);
183+
async cropImage(bucket: string, path: string, width: number, height: number, _newPath?: string) {
184+
const newPath = _newPath || `${path}_${width}_${height}`;
183185
const resizedImagePath = resolve(
184186
StorageAdapter.TEMPORARY_DIR,
185187
encodeURIComponent(join(bucket, newPath))

‎apps/nestjs-backend/src/features/attachments/plugins/s3.ts

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ import { IStorageConfig, StorageConfig } from '../../../configs/storage';
1818
import { second } from '../../../utils/second';
1919
import StorageAdapter from './adapter';
2020
import type { IPresignParams, IPresignRes, IObjectMeta, IRespHeaders } from './types';
21-
import { generateCutImagePath } from './utils';
2221

2322
@Injectable()
2423
export class S3Storage implements StorageAdapter {
@@ -148,12 +147,15 @@ export class S3Storage implements StorageAdapter {
148147
height,
149148
};
150149
}
151-
getPreviewUrl(
150+
async getPreviewUrl(
152151
bucket: string,
153152
path: string,
154153
expiresIn: number = second(this.config.urlExpireIn),
155154
respHeaders?: IRespHeaders
156-
): Promise<string> {
155+
): Promise<string | undefined> {
156+
if (!(await this.fileExists(bucket, path))) {
157+
return;
158+
}
157159
const command = new GetObjectCommand({
158160
Bucket: bucket,
159161
Key: path,
@@ -231,8 +233,8 @@ export class S3Storage implements StorageAdapter {
231233
}
232234
}
233235

234-
async cutImage(bucket: string, path: string, width: number, height: number) {
235-
const newPath = generateCutImagePath(path, width, height);
236+
async cropImage(bucket: string, path: string, width: number, height: number, _newPath?: string) {
237+
const newPath = _newPath || `${path}_${width}_${height}`;
236238
const resizedImagePath = resolve(
237239
StorageAdapter.TEMPORARY_DIR,
238240
encodeURIComponent(join(bucket, newPath))
@@ -248,10 +250,9 @@ export class S3Storage implements StorageAdapter {
248250
if (!mimetype?.startsWith('image/')) {
249251
throw new BadRequestException('Invalid image');
250252
}
251-
const metaReader = sharp();
253+
const metaReader = sharp({ failOn: 'none', unlimited: true }).resize(width, height);
252254
const sharpReader = (stream as Readable).pipe(metaReader);
253-
const resizedImage = sharpReader.resize(width, height);
254-
await resizedImage.toFile(resizedImagePath);
255+
await sharpReader.toFile(resizedImagePath);
255256
const upload = await this.uploadFileWidthPath(bucket, newPath, resizedImagePath, {
256257
'Content-Type': mimetype,
257258
});

‎apps/nestjs-backend/src/features/attachments/plugins/types.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,8 @@ export type IRespHeaders = {
3333
// eslint-disable-next-line @typescript-eslint/no-explicit-any
3434
[key: string]: any;
3535
};
36+
37+
export enum ThumbnailSize {
38+
SM = 'sm',
39+
LG = 'lg',
40+
}

‎apps/nestjs-backend/src/features/attachments/plugins/utils.ts

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { join } from 'path';
22
import { baseConfig } from '../../../configs/base.config';
33
import { storageConfig } from '../../../configs/storage';
44
import { LocalStorage } from './local';
5+
import type { ThumbnailSize } from './types';
56

67
export const getFullStorageUrl = (bucket: string, path: string) => {
78
const { storagePrefix } = baseConfig();
@@ -12,6 +13,6 @@ export const getFullStorageUrl = (bucket: string, path: string) => {
1213
return storagePrefix + join('/', bucket, path);
1314
};
1415

15-
export const generateCutImagePath = (path: string, width: number, height: number) => {
16-
return `${path}_${width}_${height}`;
16+
export const generateCropImagePath = (path: string, size: ThumbnailSize) => {
17+
return `${path}_${size}`;
1718
};

‎apps/nestjs-backend/src/features/record/open-api/record-open-api.service.ts

Lines changed: 8 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -255,8 +255,8 @@ export class RecordOpenApiService {
255255
const attachmentCv = newCellValues[i] as IAttachmentCellValue;
256256
if (field.type === FieldType.Attachment && attachmentCv) {
257257
attachmentCv.forEach((attachmentItem, index) => {
258-
const { mimetype, lgThumbnailPath, smThumbnailPath } = attachmentItem;
259-
if (mimetype.startsWith('image/') && (!lgThumbnailPath || !smThumbnailPath)) {
258+
const { mimetype } = attachmentItem;
259+
if (mimetype.startsWith('image/')) {
260260
collectionAttachmentThumbnails.push({
261261
index: i,
262262
key: fieldIdOrName,
@@ -275,22 +275,12 @@ export class RecordOpenApiService {
275275
if (!width || !height) {
276276
continue;
277277
}
278-
const { smThumbnailPath, lgThumbnailPath } =
279-
await this.attachmentsStorageService.cutTableImage(
280-
StorageAdapter.getBucket(UploadType.Table),
281-
path,
282-
width,
283-
height
284-
);
285-
attachmentItem.lgThumbnailPath = lgThumbnailPath;
286-
attachmentItem.smThumbnailPath = smThumbnailPath;
287-
const { smThumbnailUrl, lgThumbnailUrl } =
288-
await this.attachmentsStorageService.getTableAttachmentThumbnailUrl(
289-
smThumbnailPath,
290-
lgThumbnailPath
291-
);
292-
attachmentItem.smThumbnailUrl = smThumbnailUrl;
293-
attachmentItem.lgThumbnailUrl = lgThumbnailUrl;
278+
this.attachmentsStorageService.cutTableImage(
279+
StorageAdapter.getBucket(UploadType.Table),
280+
path,
281+
width,
282+
height
283+
);
294284
}
295285
}
296286
return records.map((record, i) => ({

0 commit comments

Comments
 (0)