rework redis error handling (#250)

* rework redis error handling

* wip

* wip

---------

Co-authored-by: Max Gruenfelder <[email protected]>

Author: soccermax

CHANGELOG.md

@@ -5,11 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
-## v1.7.3 - 2024-10-XX
+## v1.7.3 - 2024-11-19
 
 ### Added
 
 - allow redis mode for single tenant applications
+- error message if redis is not available during connection check
+- add option `crashOnRedisUnavailable` to crash the app if redis is not available during the connection check
 
 ## v1.7.2 - 2024-10-22
 

cds-plugin.js

@@ -4,6 +4,7 @@ const cds = require("@sap/cds");
 const cdsPackage = require("@sap/cds/package.json");
 
 const eventQueue = require("./src");
+const EventQueueError = require("./src/EventQueueError");
 const COMPONENT_NAME = "/eventQueue/plugin";
 const SERVE_COMMAND = "serve";
 
@@ -18,5 +19,10 @@ if ((doLegacyBuildDetection && isBuild) || (!doLegacyBuildDetection && !isServe)
 }
 
 if (Object.keys(cds.env.eventQueue ?? {}).length) {
-  module.exports = eventQueue.initialize().catch((err) => cds.log(COMPONENT_NAME).error(err));
+  module.exports = eventQueue.initialize().catch((err) => {
+    if (EventQueueError.isRedisConnectionFailure(err) && eventQueue.config.crashOnRedisUnavailable) {
+      throw err;
+    }
+    cds.log(COMPONENT_NAME).error(err);
+  });
 }

docs/setup/index.md

@@ -80,6 +80,7 @@ The table includes the parameter name, a description of its purpose, and the def
 | enableCAPTelemetry                   | If enabled in combination with `cap-js/telemetry`, OpenTelemetry traces about all event-queue activities are written using the `cap-js/telemetry` tracer.                                                                                                                                                                                        | false          | yes                       |
 | cronTimezone                         | Determines whether to apply the central `cronTimezone` setting for scheduling events. If set to `true`, the event will use the defined `cronTimezone`. If set to `false`, the event will use UTC or the server's local time, based on the `utc` setting.                                                                                         | null           | yes                       |
 | publishEventBlockList                | Determines whether the publication of events to all app instances is enabled when Redis is active. If set to true, events can be published; if set to false, the publication is disabled.                                                                                                                                                        | true           | yes                       |
+| crashOnRedisUnavailable              | If enabled, the application will crash if Redis is unavailable during the connection check.                                                                                                                                                                                                                                                      | false          | false                     |
 
 # Configure Redis
 

src/EventQueueError.js

@@ -41,7 +41,7 @@ const ERROR_CODES_META = {
     message: "error during create client with redis-cache service",
   },
   [ERROR_CODES.REDIS_LOCAL_NO_RECONNECT]: {
-    message: "disabled reconnect, because we are not running on cloud foundry",
+    message: "disabled reconnect, because not running on cloud foundry",
   },
   [ERROR_CODES.MISSING_TABLE_DEFINITION]: {
     message: "Could not find table in csn. Make sure the provided table name is correct and the table is known by CDS.",
@@ -135,7 +135,7 @@ class EventQueueError extends VError {
     return new EventQueueError(
       {
         name: ERROR_CODES.REDIS_CREATE_CLIENT,
-        cause: err,
+        ...(err && { cause: err }),
       },
       message
     );
@@ -325,6 +325,10 @@ class EventQueueError extends VError {
       message
     );
   }
+
+  static isRedisConnectionFailure(err) {
+    return err instanceof VError && err.name === ERROR_CODES.REDIS_CREATE_CLIENT;
+  }
 }
 
 module.exports = EventQueueError;

src/config.js

@@ -77,6 +77,7 @@ class Config {
   #unsubscribedTenants = {};
   #cronTimezone;
   #publishEventBlockList;
+  #crashOnRedisUnavailable;
   static #instance;
   constructor() {
     this.#logger = cds.log(COMPONENT_NAME);
@@ -510,6 +511,14 @@ class Config {
     this.#publishEventBlockList = value;
   }
 
+  get crashOnRedisUnavailable() {
+    return this.#crashOnRedisUnavailable;
+  }
+
+  set crashOnRedisUnavailable(value) {
+    this.#crashOnRedisUnavailable = value;
+  }
+
   set globalTxTimeout(value) {
     this.#globalTxTimeout = value;
   }

src/index.d.ts

@@ -182,6 +182,8 @@ declare class Config {
   hasEventAfterCommitFlag(type: string, subType: string): boolean;
   _checkRedisIsBound(): boolean;
   checkRedisEnabled(): boolean;
+  publishEventBlockList(): boolean;
+  crashOnRedisUnavailable(): boolean;
   attachConfigChangeHandler(): void;
   attachRedisUnsubscribeHandler(): void;
   executeUnsubscribeHandlers(tenantId: string): void;

src/initialize.js

@@ -16,6 +16,7 @@ const eventQueueAsOutbox = require("./outbox/eventQueueAsOutbox");
 const { getAllTenantIds } = require("./shared/cdsHelper");
 const { EventProcessingStatus } = require("./constants");
 const distributedLock = require("./shared/distributedLock");
+const EventQueueError = require("./EventQueueError");
 
 const readFileAsync = promisify(fs.readFile);
 
@@ -39,6 +40,7 @@ const CONFIG_VARS = [
   ["enableCAPTelemetry", false],
   ["cronTimezone", null],
   ["publishEventBlockList", true],
+  ["crashOnRedisUnavailable", false],
 ];
 
 /**
@@ -61,6 +63,7 @@ const CONFIG_VARS = [
  * @param {boolean} [options.enableCAPTelemetry=false] - Enable telemetry for CAP.
  * @param {string} [options.cronTimezone=null] - Default timezone for cron jobs.
  * @param {string} [options.publishEventBlockList=true] - If redis is available event blocklist is distributed to all application instances
+ * @param {string} [options.crashOnRedisUnavailable=true] - If enabled an error is thrown if the redis connection check is not successful
  */
 const initialize = async (options = {}) => {
   if (config.initialized) {
@@ -89,6 +92,9 @@ const initialize = async (options = {}) => {
   });
   if (redisEnabled) {
     config.redisEnabled = await redis.connectionCheck(config.redisOptions);
+    if (!config.redisEnabled && config.crashOnRedisUnavailable) {
+      throw EventQueueError.redisConnectionFailure();
+    }
   }
   config.fileContent = await readConfigFromFile(config.configFilePath);
 

src/shared/redis.js

@@ -50,25 +50,27 @@ const _createClientBase = (redisOptions) => {
   }
 };
 
-const createClientAndConnect = async (options, errorHandlerCreateClient) => {
+const createClientAndConnect = async (options, errorHandlerCreateClient, isConnectionCheck) => {
   try {
     const client = _createClientBase(options);
-    await client.connect();
-    client.on("error", (err) => {
-      const dateNow = Date.now();
-      if (dateNow - lastErrorLog > LOG_AFTER_SEC * 1000) {
-        cds.log(COMPONENT_NAME).error("error from redis client for pub/sub failed", err);
-        lastErrorLog = dateNow;
-      }
-    });
+    if (!isConnectionCheck) {
+      client.on("error", (err) => {
+        const dateNow = Date.now();
+        if (dateNow - lastErrorLog > LOG_AFTER_SEC * 1000) {
+          cds.log(COMPONENT_NAME).error("error from redis client for pub/sub failed", err);
+          lastErrorLog = dateNow;
+        }
+      });
 
-    client.on("reconnecting", () => {
-      const dateNow = Date.now();
-      if (dateNow - lastErrorLog > LOG_AFTER_SEC * 1000) {
-        cds.log(COMPONENT_NAME).info("redis client trying reconnect...");
-        lastErrorLog = dateNow;
-      }
-    });
+      client.on("reconnecting", () => {
+        const dateNow = Date.now();
+        if (dateNow - lastErrorLog > LOG_AFTER_SEC * 1000) {
+          cds.log(COMPONENT_NAME).info("redis client trying reconnect...");
+          lastErrorLog = dateNow;
+        }
+      });
+    }
+    await client.connect();
     return client;
   } catch (err) {
     errorHandlerCreateClient(err);
@@ -119,7 +121,7 @@ const _resilientClientClose = async (client) => {
 
 const connectionCheck = async (options) => {
   return new Promise((resolve, reject) => {
-    createClientAndConnect(options, reject)
+    createClientAndConnect(options, reject, true)
       .then((client) => {
         if (client) {
           _resilientClientClose(client);
@@ -131,7 +133,10 @@ const connectionCheck = async (options) => {
       .catch(reject);
   })
     .then(() => true)
-    .catch(() => false);
+    .catch((err) => {
+      cds.log(COMPONENT_NAME).error("Redis connection check failed! Falling back to NO_REDIS mode", err);
+      return false;
+    });
 };
 
 module.exports = {