feat: deviate Redis expiration time

src/config/entities/__tests__/configuration.ts

@@ -157,6 +157,7 @@ export default (): ReturnType<typeof configuration> => ({
     fromName: faker.person.fullName(),
   },
   expirationTimeInSeconds: {
+    deviatePercent: faker.number.int({ min: 10, max: 20 }),
     default: faker.number.int(),
     rpc: faker.number.int(),
     hoodi: faker.number.int(),

src/config/entities/configuration.ts

@@ -245,6 +245,7 @@ export default () => ({
     fromName: process.env.EMAIL_API_FROM_NAME || 'Safe',
   },
   expirationTimeInSeconds: {
+    deviatePercent: parseInt(process.env.EXPIRATION_DEVIATE_PERCENT ?? `${10}`),
     default: parseInt(process.env.EXPIRATION_TIME_DEFAULT_SECONDS ?? `${60}`),
     rpc: parseInt(process.env.EXPIRATION_TIME_RPC_SECONDS ?? `${15}`),
     hoodi: parseInt(process.env.HOODI_EXPIRATION_TIME_SECONDS ?? `${60}`),

src/datasources/cache/redis.cache.service.key-prefix.spec.ts

@@ -34,15 +34,20 @@ const mockConfigurationService = jest.mocked(configurationService);
 describe('RedisCacheService with a Key Prefix', () => {
   let redisCacheService: RedisCacheService;
   let defaultExpirationTimeInSeconds: number;
+  let defaultExpirationDeviatePercent: number;
   const keyPrefix = faker.string.uuid();
 
   beforeEach(() => {
     clearAllMocks();
     defaultExpirationTimeInSeconds = faker.number.int({ min: 1, max: 3600 });
+    defaultExpirationDeviatePercent = faker.number.int({ min: 1, max: 3600 });
     mockConfigurationService.getOrThrow.mockImplementation((key) => {
       if (key === 'expirationTimeInSeconds.default') {
         return defaultExpirationTimeInSeconds;
       }
+      if (key === 'expirationTimeInSeconds.deviatePercent') {
+        return defaultExpirationDeviatePercent;
+      }
       throw Error(`Unexpected key: ${key}`);
     });
 
@@ -62,7 +67,7 @@ describe('RedisCacheService with a Key Prefix', () => {
     const value = fakeJson();
     const expireTime = faker.number.int();
 
-    await redisCacheService.hSet(cacheDir, value, expireTime);
+    await redisCacheService.hSet(cacheDir, value, expireTime, 0);
 
     expect(redisClientTypeMock.hSet).toHaveBeenCalledWith(
       `${keyPrefix}-${cacheDir.key}`,

src/datasources/cache/redis.cache.service.spec.ts

@@ -8,6 +8,7 @@ import type { IConfigurationService } from '@/config/configuration.service.inter
 import clearAllMocks = jest.clearAllMocks;
 import { redisClientFactory } from '@/__tests__/redis-client.factory';
 import { MAX_TTL } from '@/datasources/cache/constants';
+import { offsetByPercentage } from '@/domain/common/utils/number';
 
 const mockLoggingService: jest.MockedObjectDeep<ILoggingService> = {
   info: jest.fn(),
@@ -24,6 +25,8 @@ const mockConfigurationService = jest.mocked(configurationService);
 describe('RedisCacheService', () => {
   let redisCacheService: RedisCacheService;
   let defaultExpirationTimeInSeconds: number;
+  let defaultExpirationDeviatePercent: number;
+  let maxTtlDeviated: number;
   const keyPrefix = '';
   let redisClient: RedisClientType;
 
@@ -39,10 +42,16 @@ describe('RedisCacheService', () => {
     clearAllMocks();
     await redisClient.flushDb();
     defaultExpirationTimeInSeconds = faker.number.int({ min: 1, max: 3600 });
+    defaultExpirationDeviatePercent = faker.number.int({ min: 1, max: 99 });
+    maxTtlDeviated =
+      MAX_TTL - (MAX_TTL * defaultExpirationDeviatePercent) / 100;
     mockConfigurationService.getOrThrow.mockImplementation((key) => {
       if (key === 'expirationTimeInSeconds.default') {
         return defaultExpirationTimeInSeconds;
       }
+      if (key === 'expirationTimeInSeconds.deviatePercent') {
+        return defaultExpirationDeviatePercent;
+      }
       throw Error(`Unexpected key: ${key}`);
     });
 
@@ -75,7 +84,7 @@ describe('RedisCacheService', () => {
     const value = fakeJson();
     const expireTime = faker.number.int();
 
-    await redisCacheService.hSet(cacheDir, value, expireTime);
+    await redisCacheService.hSet(cacheDir, value, expireTime, 0);
 
     const storedValue = await redisClient.hGet(cacheDir.key, cacheDir.field);
     const ttl = await redisClient.ttl(cacheDir.key);
@@ -84,19 +93,48 @@ describe('RedisCacheService', () => {
     expect(ttl).toBeLessThanOrEqual(expireTime);
   });
 
-  it('Setting key throws on expire', async () => {
+  it('Setting key with expireTimeSeconds and expireDeviatePercent does store the value with the deviated TTL', async () => {
     const cacheDir = new CacheDir(
       faker.string.alphanumeric(),
       faker.string.sample(),
     );
+    const value = fakeJson();
+    const expireTime = faker.number.int({ min: 1, max: 3600 });
+    const expireDeviatePercent = faker.number.int({ min: 1, max: 100 });
+    const maxDeviation = offsetByPercentage(expireTime, expireDeviatePercent);
+
+    await redisCacheService.hSet(
+      cacheDir,
+      value,
+      expireTime,
+      expireDeviatePercent,
+    );
 
-    // Expiration time out of range to force an error
-    await expect(
-      redisCacheService.hSet(cacheDir, '', Number.MAX_VALUE + 1),
-    ).rejects.toThrow();
+    const storedValue = await redisClient.hGet(cacheDir.key, cacheDir.field);
+    const ttl = await redisClient.ttl(cacheDir.key);
+    expect(storedValue).toEqual(value);
+    expect(ttl).toBeGreaterThan(0);
+    expect(ttl).toBeLessThanOrEqual(maxDeviation);
+  });
+
+  it('Setting key with expireTimeSeconds and no expireDeviatePercent does store the value with the default TTL deviation', async () => {
+    const cacheDir = new CacheDir(
+      faker.string.alphanumeric(),
+      faker.string.sample(),
+    );
+    const value = fakeJson();
+    const expireTime = faker.number.int({ min: 1, max: 3600 });
+    const maxDeviation = offsetByPercentage(
+      expireTime,
+      defaultExpirationDeviatePercent,
+    );
+    await redisCacheService.hSet(cacheDir, value, expireTime);
 
     const storedValue = await redisClient.hGet(cacheDir.key, cacheDir.field);
-    expect(storedValue).toBeNull();
+    const ttl = await redisClient.ttl(cacheDir.key);
+    expect(storedValue).toEqual(value);
+    expect(ttl).toBeGreaterThan(0);
+    expect(ttl).toBeLessThanOrEqual(maxDeviation);
   });
 
   it('Getting key gets the stored value', async () => {
@@ -148,19 +186,27 @@ describe('RedisCacheService', () => {
   });
 
   it('creates a missing key and increments its value', async () => {
-    const expireTime = faker.number.int({ min: 1 });
+    const expireTime = faker.number.int({ min: 1, max: maxTtlDeviated });
+    const maxExpireTime = offsetByPercentage(
+      expireTime,
+      defaultExpirationDeviatePercent,
+    );
     const key = faker.string.alphanumeric();
 
     const firstResult = await redisCacheService.increment(key, expireTime);
 
     const ttl = await redisClient.ttl(key);
     expect(firstResult).toEqual(1);
     expect(ttl).toBeGreaterThan(0);
-    expect(ttl).toBeLessThanOrEqual(expireTime);
+    expect(ttl).toBeLessThanOrEqual(maxExpireTime);
   });
 
   it('increments the value of an existing key', async () => {
-    const expireTime = faker.number.int({ min: 1 });
+    const expireTime = faker.number.int({ min: 1, max: maxTtlDeviated });
+    const maxExpireTime = offsetByPercentage(
+      expireTime,
+      defaultExpirationDeviatePercent,
+    );
     const key = faker.string.alphanumeric();
     const initialValue = faker.number.int({ min: 100 });
     await redisClient.set(key, initialValue, { EX: expireTime });
@@ -172,21 +218,21 @@ describe('RedisCacheService', () => {
 
     const ttl = await redisClient.ttl(key);
     expect(ttl).toBeGreaterThan(0);
-    expect(ttl).toBeLessThanOrEqual(expireTime);
+    expect(ttl).toBeLessThanOrEqual(maxExpireTime);
   });
 
   it('sets and gets the value of a counter key', async () => {
     const key = faker.string.alphanumeric();
     const value = faker.number.int({ min: 100 });
-    await redisCacheService.setCounter(key, value, MAX_TTL);
+    await redisCacheService.setCounter(key, value, maxTtlDeviated);
 
     const result = await redisCacheService.getCounter(key);
     expect(result).toEqual(value);
   });
 
   it('sets and gets the value of a zero-value counter', async () => {
     const key = faker.string.alphanumeric();
-    await redisCacheService.setCounter(key, 0, MAX_TTL);
+    await redisCacheService.setCounter(key, 0, maxTtlDeviated);
 
     const result = await redisCacheService.getCounter(key);
     expect(result).toEqual(0);
@@ -225,7 +271,7 @@ describe('RedisCacheService', () => {
     const value = faker.string.sample();
 
     try {
-      await redisCacheService.hSet(new CacheDir(key, ''), value, MAX_TTL);
+      await redisCacheService.hSet(new CacheDir(key, ''), value, MAX_TTL, 0);
     } catch (err) {
       console.error(err);
       throw new Error('Should not throw');
@@ -237,4 +283,44 @@ describe('RedisCacheService', () => {
     expect(ttl).toBeGreaterThan(0);
     expect(ttl).toBeLessThanOrEqual(Number.MAX_SAFE_INTEGER);
   });
+
+  it('Setting key with TTL larger than MAX_TTL enforces MAX_TTL limit', async () => {
+    const cacheDir = new CacheDir(
+      faker.string.alphanumeric(),
+      faker.string.sample(),
+    );
+    const value = fakeJson();
+    const expireTime = MAX_TTL + faker.number.int({ min: 1000, max: 10000 });
+
+    await redisCacheService.hSet(cacheDir, value, expireTime, 0);
+
+    const storedValue = await redisClient.hGet(cacheDir.key, cacheDir.field);
+    const ttl = await redisClient.ttl(cacheDir.key);
+    expect(storedValue).toEqual(value);
+    expect(ttl).toBeGreaterThan(0);
+    expect(ttl).toBeLessThanOrEqual(MAX_TTL);
+  });
+
+  it('Increment with TTL larger than MAX_TTL enforces MAX_TTL limit', async () => {
+    const key = faker.string.alphanumeric();
+    const expireTime = MAX_TTL + faker.number.int({ min: 1000, max: 10000 });
+
+    await redisCacheService.increment(key, expireTime, 0);
+
+    const ttl = await redisClient.ttl(key);
+    expect(ttl).toBeGreaterThan(0);
+    expect(ttl).toBeLessThanOrEqual(MAX_TTL);
+  });
+
+  it('SetCounter with TTL larger than MAX_TTL enforces MAX_TTL limit', async () => {
+    const key = faker.string.alphanumeric();
+    const value = faker.number.int({ min: 1, max: 100 });
+    const expireTime = MAX_TTL + faker.number.int({ min: 1000, max: 10000 });
+
+    await redisCacheService.setCounter(key, value, expireTime, 0);
+
+    const ttl = await redisClient.ttl(key);
+    expect(ttl).toBeGreaterThan(0);
+    expect(ttl).toBeLessThanOrEqual(MAX_TTL);
+  });
 });

src/datasources/cache/redis.cache.service.ts

@@ -5,15 +5,17 @@ import { CacheDir } from '@/datasources/cache/entities/cache-dir.entity';
 import { ICacheReadiness } from '@/domain/interfaces/cache-readiness.interface';
 import { ILoggingService, LoggingService } from '@/logging/logging.interface';
 import { IConfigurationService } from '@/config/configuration.service.interface';
-import { CacheKeyPrefix } from '@/datasources/cache/constants';
+import { CacheKeyPrefix, MAX_TTL } from '@/datasources/cache/constants';
 import { LogType } from '@/domain/common/entities/log-type.entity';
+import { deviateRandomlyByPercentage } from '@/domain/common/utils/number';
 
 @Injectable()
 export class RedisCacheService
   implements ICacheService, ICacheReadiness, OnModuleDestroy
 {
   private readonly quitTimeoutInSeconds: number = 2;
   private readonly defaultExpirationTimeInSeconds: number;
+  private readonly defaultExpirationDeviatePercent: number;
 
   constructor(
     @Inject('RedisClient') private readonly client: RedisClientType,
@@ -26,6 +28,10 @@ export class RedisCacheService
       this.configurationService.getOrThrow<number>(
         'expirationTimeInSeconds.default',
       );
+    this.defaultExpirationDeviatePercent =
+      this.configurationService.getOrThrow<number>(
+        'expirationTimeInSeconds.deviatePercent',
+      );
   }
 
   async ping(): Promise<unknown> {
@@ -46,18 +52,25 @@ export class RedisCacheService
     cacheDir: CacheDir,
     value: string,
     expireTimeSeconds: number | undefined,
+    expireDeviatePercent?: number,
   ): Promise<void> {
     if (!expireTimeSeconds || expireTimeSeconds <= 0) {
       return;
     }
 
     const key = this._prefixKey(cacheDir.key);
+    const expirationTime = this.enforceMaxRedisTTL(
+      deviateRandomlyByPercentage(
+        expireTimeSeconds,
+        expireDeviatePercent ?? this.defaultExpirationDeviatePercent,
+      ),
+    );
 
     try {
       await this.client.hSet(key, cacheDir.field, value);
       // NX - Set expiry only when the key has no expiry
       // See https://redis.io/commands/expire/
-      await this.client.expire(key, expireTimeSeconds, 'NX');
+      await this.client.expire(key, expirationTime, 'NX');
     } catch (error) {
       this.loggingService.error({
         type: LogType.CacheError,
@@ -78,21 +91,31 @@ export class RedisCacheService
     const keyWithPrefix = this._prefixKey(key);
     // see https://redis.io/commands/unlink/
     const result = await this.client.unlink(keyWithPrefix);
+
     await this.hSet(
       new CacheDir(`invalidationTimeMs:${key}`, ''),
       Date.now().toString(),
       this.defaultExpirationTimeInSeconds,
+      0,
     );
     return result;
   }
 
   async increment(
     cacheKey: string,
     expireTimeSeconds: number | undefined,
+    expireDeviatePercent?: number,
   ): Promise<number> {
     const transaction = this.client.multi().incr(cacheKey);
     if (expireTimeSeconds !== undefined && expireTimeSeconds > 0) {
-      transaction.expire(cacheKey, expireTimeSeconds, 'NX');
+      const expirationTime = this.enforceMaxRedisTTL(
+        deviateRandomlyByPercentage(
+          expireTimeSeconds,
+          expireDeviatePercent ?? this.defaultExpirationDeviatePercent,
+        ),
+      );
+
+      transaction.expire(cacheKey, expirationTime, 'NX');
     }
     const [incrRes] = await transaction.get(cacheKey).exec();
     return Number(incrRes);
@@ -102,8 +125,19 @@ export class RedisCacheService
     key: string,
     value: number,
     expireTimeSeconds: number,
+    expireDeviatePercent?: number,
   ): Promise<void> {
-    await this.client.set(key, value, { EX: expireTimeSeconds, NX: true });
+    const expirationTime = this.enforceMaxRedisTTL(
+      deviateRandomlyByPercentage(
+        expireTimeSeconds,
+        expireDeviatePercent ?? this.defaultExpirationDeviatePercent,
+      ),
+    );
+
+    await this.client.set(key, value, {
+      EX: expirationTime,
+      NX: true,
+    });
   }
 
   /**
@@ -154,4 +188,15 @@ export class RedisCacheService
     });
     await this.client.disconnect();
   }
+
+  /**
+   * Enforces the maximum TTL for Redis to prevent overflow errors.
+   *
+   * @param {number} ttl - The TTL to enforce.
+   *
+   * @returns {number} The TTL if it is less than or equal to MAX_TTL, otherwise MAX_TTL.
+   */
+  private enforceMaxRedisTTL(ttl: number): number {
+    return ttl > MAX_TTL ? MAX_TTL : ttl;
+  }
 }