feat: deviate Redis expiration time (#2608)

PooyaRaki · web-flow · commit 35505d0b4af9 · 2025-06-05T17:07:54.000+02:00
This PR introduces a random deviation to Redis key expiration times to prevent cache stampedes and reduce the likelihood of simultaneous expirations. By spreading out key expirations, we help distribute network load more evenly and avoid putting unnecessary pressure on other services like the TX service.
diff --git a/src/config/configuration.validator.spec.ts b/src/config/configuration.validator.spec.ts
@@ -22,6 +22,7 @@ describe('Configuration validator', () => {
     EMAIL_TEMPLATE_RECOVERY_TX: faker.string.alphanumeric(),
     EMAIL_TEMPLATE_UNKNOWN_RECOVERY_TX: faker.string.alphanumeric(),
     EMAIL_TEMPLATE_VERIFICATION_CODE: faker.string.alphanumeric(),
+    EXPIRATION_DEVIATE_PERCENT: faker.number.int({ min: 0, max: 100 }),
     FINGERPRINT_ENCRYPTION_KEY: faker.string.uuid(),
     INFURA_API_KEY: faker.string.uuid(),
     JWT_ISSUER: faker.string.uuid(),
@@ -118,6 +119,7 @@ describe('Configuration validator', () => {
       EMAIL_TEMPLATE_RECOVERY_TX: faker.string.alphanumeric(),
       EMAIL_TEMPLATE_UNKNOWN_RECOVERY_TX: faker.string.alphanumeric(),
       EMAIL_TEMPLATE_VERIFICATION_CODE: faker.string.alphanumeric(),
+      EXPIRATION_DEVIATE_PERCENT: faker.number.int({ min: 0, max: 100 }),
       FINGERPRINT_ENCRYPTION_KEY: faker.string.uuid(),
       INFURA_API_KEY: faker.string.uuid(),
       JWT_ISSUER: faker.string.uuid(),
@@ -168,6 +170,7 @@ describe('Configuration validator', () => {
       EMAIL_TEMPLATE_RECOVERY_TX: faker.string.alphanumeric(),
       EMAIL_TEMPLATE_UNKNOWN_RECOVERY_TX: faker.string.alphanumeric(),
       EMAIL_TEMPLATE_VERIFICATION_CODE: faker.string.alphanumeric(),
+      EXPIRATION_DEVIATE_PERCENT: faker.number.int({ min: 0, max: 100 }),
       FINGERPRINT_ENCRYPTION_KEY: faker.string.uuid(),
       INFURA_API_KEY: faker.string.uuid(),
       JWT_ISSUER: faker.string.uuid(),
diff --git a/src/config/entities/__tests__/configuration.ts b/src/config/entities/__tests__/configuration.ts
@@ -157,6 +157,7 @@ export default (): ReturnType<typeof configuration> => ({
     fromName: faker.person.fullName(),
   },
   expirationTimeInSeconds: {
+    deviatePercent: faker.number.int({ min: 10, max: 20 }),
     default: faker.number.int(),
     rpc: faker.number.int(),
     hoodi: faker.number.int(),
diff --git a/src/config/entities/configuration.ts b/src/config/entities/configuration.ts
@@ -245,6 +245,7 @@ export default () => ({
     fromName: process.env.EMAIL_API_FROM_NAME || 'Safe',
   },
   expirationTimeInSeconds: {
+    deviatePercent: parseInt(process.env.EXPIRATION_DEVIATE_PERCENT ?? `${10}`),
     default: parseInt(process.env.EXPIRATION_TIME_DEFAULT_SECONDS ?? `${60}`),
     rpc: parseInt(process.env.EXPIRATION_TIME_RPC_SECONDS ?? `${15}`),
     hoodi: parseInt(process.env.HOODI_EXPIRATION_TIME_SECONDS ?? `${60}`),
diff --git a/src/config/entities/schemas/configuration.schema.ts b/src/config/entities/schemas/configuration.schema.ts
@@ -19,6 +19,11 @@ export const RootConfigurationSchema = z
     EMAIL_TEMPLATE_RECOVERY_TX: z.string(),
     EMAIL_TEMPLATE_UNKNOWN_RECOVERY_TX: z.string(),
     EMAIL_TEMPLATE_VERIFICATION_CODE: z.string(),
+    EXPIRATION_DEVIATE_PERCENT: z
+      .number({ coerce: true })
+      .min(0)
+      .max(100)
+      .optional(),
     FINGERPRINT_ENCRYPTION_KEY: z.string(),
     INFURA_API_KEY: z.string(),
     JWT_ISSUER: z.string(),
diff --git a/src/datasources/cache/redis.cache.service.key-prefix.spec.ts b/src/datasources/cache/redis.cache.service.key-prefix.spec.ts
@@ -34,15 +34,20 @@ const mockConfigurationService = jest.mocked(configurationService);
 describe('RedisCacheService with a Key Prefix', () => {
   let redisCacheService: RedisCacheService;
   let defaultExpirationTimeInSeconds: number;
+  let defaultExpirationDeviatePercent: number;
   const keyPrefix = faker.string.uuid();
 
   beforeEach(() => {
     clearAllMocks();
     defaultExpirationTimeInSeconds = faker.number.int({ min: 1, max: 3600 });
+    defaultExpirationDeviatePercent = faker.number.int({ min: 1, max: 3600 });
     mockConfigurationService.getOrThrow.mockImplementation((key) => {
       if (key === 'expirationTimeInSeconds.default') {
         return defaultExpirationTimeInSeconds;
       }
+      if (key === 'expirationTimeInSeconds.deviatePercent') {
+        return defaultExpirationDeviatePercent;
+      }
       throw Error(`Unexpected key: ${key}`);
     });
 
@@ -62,7 +67,7 @@ describe('RedisCacheService with a Key Prefix', () => {
     const value = fakeJson();
     const expireTime = faker.number.int();
 
-    await redisCacheService.hSet(cacheDir, value, expireTime);
+    await redisCacheService.hSet(cacheDir, value, expireTime, 0);
 
     expect(redisClientTypeMock.hSet).toHaveBeenCalledWith(
       `${keyPrefix}-${cacheDir.key}`,
diff --git a/src/datasources/cache/redis.cache.service.spec.ts b/src/datasources/cache/redis.cache.service.spec.ts
@@ -8,6 +8,7 @@ import type { IConfigurationService } from '@/config/configuration.service.inter
 import clearAllMocks = jest.clearAllMocks;
 import { redisClientFactory } from '@/__tests__/redis-client.factory';
 import { MAX_TTL } from '@/datasources/cache/constants';
+import { offsetByPercentage } from '@/domain/common/utils/number';
 
 const mockLoggingService: jest.MockedObjectDeep<ILoggingService> = {
   info: jest.fn(),
@@ -24,6 +25,8 @@ const mockConfigurationService = jest.mocked(configurationService);
 describe('RedisCacheService', () => {
   let redisCacheService: RedisCacheService;
   let defaultExpirationTimeInSeconds: number;
+  let defaultExpirationDeviatePercent: number;
+  let maxTtlDeviated: number;
   const keyPrefix = '';
   let redisClient: RedisClientType;
 
@@ -39,10 +42,16 @@ describe('RedisCacheService', () => {
     clearAllMocks();
     await redisClient.flushDb();
     defaultExpirationTimeInSeconds = faker.number.int({ min: 1, max: 3600 });
+    defaultExpirationDeviatePercent = faker.number.int({ min: 1, max: 99 });
+    maxTtlDeviated =
+      MAX_TTL - (MAX_TTL * defaultExpirationDeviatePercent) / 100;
     mockConfigurationService.getOrThrow.mockImplementation((key) => {
       if (key === 'expirationTimeInSeconds.default') {
         return defaultExpirationTimeInSeconds;
       }
+      if (key === 'expirationTimeInSeconds.deviatePercent') {
+        return defaultExpirationDeviatePercent;
+      }
       throw Error(`Unexpected key: ${key}`);
     });
 
@@ -75,7 +84,7 @@ describe('RedisCacheService', () => {
     const value = fakeJson();
     const expireTime = faker.number.int();
 
-    await redisCacheService.hSet(cacheDir, value, expireTime);
+    await redisCacheService.hSet(cacheDir, value, expireTime, 0);
 
     const storedValue = await redisClient.hGet(cacheDir.key, cacheDir.field);
     const ttl = await redisClient.ttl(cacheDir.key);
@@ -84,19 +93,48 @@ describe('RedisCacheService', () => {
     expect(ttl).toBeLessThanOrEqual(expireTime);
   });
 
-  it('Setting key throws on expire', async () => {
+  it('Setting key with expireTimeSeconds and expireDeviatePercent does store the value with the deviated TTL', async () => {
     const cacheDir = new CacheDir(
       faker.string.alphanumeric(),
       faker.string.sample(),
     );
+    const value = fakeJson();
+    const expireTime = faker.number.int({ min: 1, max: 3600 });
+    const expireDeviatePercent = faker.number.int({ min: 1, max: 100 });
+    const maxDeviation = offsetByPercentage(expireTime, expireDeviatePercent);
+
+    await redisCacheService.hSet(
+      cacheDir,
+      value,
+      expireTime,
+      expireDeviatePercent,
+    );
 
-    // Expiration time out of range to force an error
-    await expect(
-      redisCacheService.hSet(cacheDir, '', Number.MAX_VALUE + 1),
-    ).rejects.toThrow();
+    const storedValue = await redisClient.hGet(cacheDir.key, cacheDir.field);
+    const ttl = await redisClient.ttl(cacheDir.key);
+    expect(storedValue).toEqual(value);
+    expect(ttl).toBeGreaterThan(0);
+    expect(ttl).toBeLessThanOrEqual(maxDeviation);
+  });
+
+  it('Setting key with expireTimeSeconds and no expireDeviatePercent does store the value with the default TTL deviation', async () => {
+    const cacheDir = new CacheDir(
+      faker.string.alphanumeric(),
+      faker.string.sample(),
+    );
+    const value = fakeJson();
+    const expireTime = faker.number.int({ min: 1, max: 3600 });
+    const maxDeviation = offsetByPercentage(
+      expireTime,
+      defaultExpirationDeviatePercent,
+    );
+    await redisCacheService.hSet(cacheDir, value, expireTime);
 
     const storedValue = await redisClient.hGet(cacheDir.key, cacheDir.field);
-    expect(storedValue).toBeNull();
+    const ttl = await redisClient.ttl(cacheDir.key);
+    expect(storedValue).toEqual(value);
+    expect(ttl).toBeGreaterThan(0);
+    expect(ttl).toBeLessThanOrEqual(maxDeviation);
   });
 
   it('Getting key gets the stored value', async () => {
@@ -148,19 +186,27 @@ describe('RedisCacheService', () => {
   });
 
   it('creates a missing key and increments its value', async () => {
-    const expireTime = faker.number.int({ min: 1 });
+    const expireTime = faker.number.int({ min: 1, max: maxTtlDeviated });
+    const maxExpireTime = offsetByPercentage(
+      expireTime,
+      defaultExpirationDeviatePercent,
+    );
     const key = faker.string.alphanumeric();
 
     const firstResult = await redisCacheService.increment(key, expireTime);
 
     const ttl = await redisClient.ttl(key);
     expect(firstResult).toEqual(1);
     expect(ttl).toBeGreaterThan(0);
-    expect(ttl).toBeLessThanOrEqual(expireTime);
+    expect(ttl).toBeLessThanOrEqual(maxExpireTime);
   });
 
   it('increments the value of an existing key', async () => {
-    const expireTime = faker.number.int({ min: 1 });
+    const expireTime = faker.number.int({ min: 1, max: maxTtlDeviated });
+    const maxExpireTime = offsetByPercentage(
+      expireTime,
+      defaultExpirationDeviatePercent,
+    );
     const key = faker.string.alphanumeric();
     const initialValue = faker.number.int({ min: 100 });
     await redisClient.set(key, initialValue, { EX: expireTime });
@@ -172,21 +218,21 @@ describe('RedisCacheService', () => {
 
     const ttl = await redisClient.ttl(key);
     expect(ttl).toBeGreaterThan(0);
-    expect(ttl).toBeLessThanOrEqual(expireTime);
+    expect(ttl).toBeLessThanOrEqual(maxExpireTime);
   });
 
   it('sets and gets the value of a counter key', async () => {
     const key = faker.string.alphanumeric();
     const value = faker.number.int({ min: 100 });
-    await redisCacheService.setCounter(key, value, MAX_TTL);
+    await redisCacheService.setCounter(key, value, maxTtlDeviated);
 
     const result = await redisCacheService.getCounter(key);
     expect(result).toEqual(value);
   });
 
   it('sets and gets the value of a zero-value counter', async () => {
     const key = faker.string.alphanumeric();
-    await redisCacheService.setCounter(key, 0, MAX_TTL);
+    await redisCacheService.setCounter(key, 0, maxTtlDeviated);
 
     const result = await redisCacheService.getCounter(key);
     expect(result).toEqual(0);
@@ -225,7 +271,7 @@ describe('RedisCacheService', () => {
     const value = faker.string.sample();
 
     try {
-      await redisCacheService.hSet(new CacheDir(key, ''), value, MAX_TTL);
+      await redisCacheService.hSet(new CacheDir(key, ''), value, MAX_TTL, 0);
     } catch (err) {
       console.error(err);
       throw new Error('Should not throw');
@@ -237,4 +283,44 @@ describe('RedisCacheService', () => {
     expect(ttl).toBeGreaterThan(0);
     expect(ttl).toBeLessThanOrEqual(Number.MAX_SAFE_INTEGER);
   });
+
+  it('Setting key with TTL larger than MAX_TTL enforces MAX_TTL limit', async () => {
+    const cacheDir = new CacheDir(
+      faker.string.alphanumeric(),
+      faker.string.sample(),
+    );
+    const value = fakeJson();
+    const expireTime = MAX_TTL + faker.number.int({ min: 1000, max: 10000 });
+
+    await redisCacheService.hSet(cacheDir, value, expireTime, 0);
+
+    const storedValue = await redisClient.hGet(cacheDir.key, cacheDir.field);
+    const ttl = await redisClient.ttl(cacheDir.key);
+    expect(storedValue).toEqual(value);
+    expect(ttl).toBeGreaterThan(0);
+    expect(ttl).toBeLessThanOrEqual(MAX_TTL);
+  });
+
+  it('Increment with TTL larger than MAX_TTL enforces MAX_TTL limit', async () => {
+    const key = faker.string.alphanumeric();
+    const expireTime = MAX_TTL + faker.number.int({ min: 1000, max: 10000 });
+
+    await redisCacheService.increment(key, expireTime, 0);
+
+    const ttl = await redisClient.ttl(key);
+    expect(ttl).toBeGreaterThan(0);
+    expect(ttl).toBeLessThanOrEqual(MAX_TTL);
+  });
+
+  it('SetCounter with TTL larger than MAX_TTL enforces MAX_TTL limit', async () => {
+    const key = faker.string.alphanumeric();
+    const value = faker.number.int({ min: 1, max: 100 });
+    const expireTime = MAX_TTL + faker.number.int({ min: 1000, max: 10000 });
+
+    await redisCacheService.setCounter(key, value, expireTime, 0);
+
+    const ttl = await redisClient.ttl(key);
+    expect(ttl).toBeGreaterThan(0);
+    expect(ttl).toBeLessThanOrEqual(MAX_TTL);
+  });
 });
diff --git a/src/datasources/cache/redis.cache.service.ts b/src/datasources/cache/redis.cache.service.ts
diff --git a/src/domain/common/utils/__tests__/number.spec.ts b/src/domain/common/utils/__tests__/number.spec.ts
diff --git a/src/domain/common/utils/number.ts b/src/domain/common/utils/number.ts
