util-logopenfile: Add log rotation thread to rotate during zero-traffic periods

src/util-logopenfile.c

@@ -53,6 +53,12 @@ static bool LogFileNewThreadedCtx(LogFileCtx *parent_ctx, const char *log_path,
 // Threaded eve.json identifier
 static SC_ATOMIC_DECL_AND_INIT_WITH_VAL(uint16_t, eve_file_id, 1);
 
+// Time-based log rotation globals
+static pthread_t time_rotation_thread;
+static SCMutex time_rotation_mutex = SCMUTEX_INITIALIZER;
+static LogFileCtx *time_rotation_list = NULL;
+static volatile bool time_rotation_running = false;
+
 #ifdef BUILD_WITH_UNIXSOCKET
 /** \brief connect to the indicated local stream socket, logging any errors
  *  \param path filesystem path to connect to
@@ -206,31 +212,48 @@ static void SCLogFileFlush(LogFileCtx *log_ctx)
 }
 
 /**
- * \brief Write buffer to log file.
- * \retval 0 on failure; otherwise, the return value of fwrite_unlocked (number of
- * characters successfully written).
+ * \brief Handle log file rotation checks and updates
+ * \param log_ctx Log file context
+ * \retval true if rotation occurred
  */
-static int SCLogFileWriteNoLock(const char *buffer, int buffer_len, LogFileCtx *log_ctx)
+static bool HandleLogRotation(LogFileCtx *log_ctx)
 {
-    int ret = 0;
+    bool need_rotation = false;
+    time_t now = time(NULL);
 
-    DEBUG_VALIDATE_BUG_ON(log_ctx->is_sock);
-
-    /* Check for rotation. */
     if (log_ctx->rotation_flag) {
         log_ctx->rotation_flag = 0;
-        SCConfLogReopen(log_ctx);
+        need_rotation = true;
     }
 
-    if (log_ctx->flags & LOGFILE_ROTATE_INTERVAL) {
-        time_t now = time(NULL);
-        if (now >= log_ctx->rotate_time) {
-            SCConfLogReopen(log_ctx);
+    if ((log_ctx->flags & LOGFILE_ROTATE_INTERVAL) && now >= log_ctx->rotate_time) {
+        need_rotation = true;
+    }
+
+    if (need_rotation) {
+        SCConfLogReopen(log_ctx);
+        if (log_ctx->flags & LOGFILE_ROTATE_INTERVAL) {
             log_ctx->rotate_time = now + log_ctx->rotate_interval;
         }
     }
 
-    if (log_ctx->fp) {
+    return need_rotation;
+}
+
+/**
+ * \brief Write buffer to log file.
+ * \retval 0 on failure; otherwise, the return value of fwrite_unlocked (number of
+ * characters successfully written).
+ */
+static int SCLogFileWriteNoLock(const char *buffer, int buffer_len, LogFileCtx *log_ctx)
+{
+    int ret = 0;
+
+    DEBUG_VALIDATE_BUG_ON(log_ctx->is_sock);
+
+    HandleLogRotation(log_ctx);
+
+    if (buffer_len > 0 && log_ctx->fp) {
         SCClearErrUnlocked(log_ctx->fp);
         if (1 != SCFwriteUnlocked(buffer, buffer_len, 1, log_ctx->fp)) {
             /* Only the first error is logged */
@@ -394,6 +417,48 @@ bool SCLogOpenThreadedFile(const char *log_path, const char *append, LogFileCtx
         return false;
 }
 
+/** \brief Triggers rotation checks during zero-traffic periods
+ *  \param arg not used
+ */
+static void *TimeBasedLogRotationThread(void *arg)
+{
+    while (time_rotation_running) {
+        sleep(1);
+
+        SCMutexLock(&time_rotation_mutex);
+        LogFileCtx *ctx = time_rotation_list;
+
+        while (ctx != NULL) {
+            ctx->Write("", 0, ctx);
+            ctx = ctx->time_rotation_next;
+        }
+
+        SCMutexUnlock(&time_rotation_mutex);
+    }
+    return NULL;
+}
+
+static void RegisterTimeBasedLogRotation(LogFileCtx *log_ctx)
+{
+    if (!log_ctx->is_regular) {
+        return;
+    }
+
+    SCMutexLock(&time_rotation_mutex);
+
+    if (!time_rotation_running) {
+        time_rotation_running = true;
+        if (pthread_create(&time_rotation_thread, NULL, TimeBasedLogRotationThread, NULL) != 0) {
+            FatalError("Failed to create time-based log rotation thread");
+        }
+    }
+
+    log_ctx->time_rotation_next = time_rotation_list;
+    time_rotation_list = log_ctx;
+
+    SCMutexUnlock(&time_rotation_mutex);
+}
+
 /** \brief open the indicated file, logging any errors
  *  \param path filesystem path to open
  *  \param append_setting open file with O_APPEND: "yes" or "no"
@@ -637,6 +702,7 @@ int SCConfLogOpenGeneric(
         SCLogError("Failed to allocate memory for filename");
         return -1;
     }
+    RegisterTimeBasedLogRotation(log_ctx);
 
 #ifdef BUILD_WITH_UNIXSOCKET
     /* If a socket and running live, do non-blocking writes. */
@@ -871,6 +937,7 @@ static bool LogFileNewThreadedCtx(LogFileCtx *parent_ctx, const char *log_path,
         thread->Write = SCLogFileWriteNoLock;
         thread->Close = SCLogFileCloseNoLock;
         OutputRegisterFileRotationFlag(&thread->rotation_flag);
+        RegisterTimeBasedLogRotation(thread);
     } else if (parent_ctx->type == LOGFILE_TYPE_FILETYPE) {
         entry->slot_number = SC_ATOMIC_ADD(eve_file_id, 1);
         SCLogDebug("%s - thread %d [slot %d]", log_path, entry->internal_thread_id,

src/util-logopenfile.h

@@ -126,6 +126,9 @@ typedef struct LogFileCtx_ {
     /** The interval to rotate the log file */
     uint64_t rotate_interval;
 
+    /** Next context in time-based rotation list */
+    struct LogFileCtx_ *time_rotation_next;
+
     /**< Used by some alert loggers like the unified ones that append
      * the date onto the end of files. */
     char *prefix;