logging: add clock-jump recovery and tighten Alloy service ordering

- Add ghaf.logging.recovery options and shared clock-jump watcher + recover oneshot.
- Ensure alloy.service is ordered after/requires systemd-journald on client and server.
- Server pipeline: route journald through loki.process, drop entries older than 168h, and align WAL max_segment_age.

Signed-off-by: Everton de Matos <[email protected]>

Author: everton-dematos

modules/common/logging/client.nix

@@ -124,19 +124,24 @@ in
 
     services.alloy.enable = true;
 
-    # Copy certs/keys (and optional CA) into /run/credentials/alloy.service/…
-    systemd.services.alloy.serviceConfig.LoadCredential = [
-      "client_cert:${cfg.tls.certFile}"
-      "client_key:${cfg.tls.keyFile}"
-    ]
-    ++ lib.optionals (cfg.tls.caFile != null) [
-      "client_ca:${cfg.tls.caFile}"
-    ];
+    systemd.services.alloy.serviceConfig = {
+      after = [ "systemd-journald.service" ];
+      requires = [ "systemd-journald.service" ];
+
+      # Once alloy.service in admin-vm stopped this service will
+      # still keep on retrying to send logs batch, so we need to
+      # stop it forcefully.
+      TimeoutStopSec = 4;
 
-    # Once alloy.service in admin-vm stopped this service will
-    # still keep on retrying to send logs batch, so we need to
-    # stop it forcefully.
-    systemd.services.alloy.serviceConfig.TimeoutStopSec = 4;
+      # Copy certs/keys (and optional CA) into /run/credentials/alloy.service/…
+      LoadCredential = [
+        "client_cert:${cfg.tls.certFile}"
+        "client_key:${cfg.tls.keyFile}"
+      ]
+      ++ lib.optionals (cfg.tls.caFile != null) [
+        "client_ca:${cfg.tls.caFile}"
+      ];
+    };
 
     ghaf.security.audit.extraRules = [
       "-w /etc/alloy/client.alloy -p rwxa -k alloy_client_config"

modules/common/logging/common.nix

@@ -1,8 +1,73 @@
 # SPDX-FileCopyrightText: 2022-2026 TII (SSRC) and the Ghaf contributors
 # SPDX-License-Identifier: Apache-2.0
-{ lib, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
 let
-  inherit (lib) mkOption types;
+  inherit (lib) mkIf mkOption types;
+  recCfg = config.ghaf.logging.recovery;
+
+  ghafClockJumpWatcher = pkgs.writeShellApplication {
+    name = "ghaf-clock-jump-watcher";
+    runtimeInputs = with pkgs; [
+      coreutils
+      gawk
+      systemd
+    ];
+    text = ''
+      threshold="${toString recCfg.thresholdSeconds}"
+      interval="${toString recCfg.intervalSeconds}"
+
+      last_real="$(date +%s)"
+      last_up="$(cut -d' ' -f1 /proc/uptime)"
+
+      while true; do
+        sleep "$interval"
+        real="$(date +%s)"
+        up="$(cut -d' ' -f1 /proc/uptime)"
+
+        drift="$(awk -v r1="$last_real" -v r2="$real" -v u1="$last_up" -v u2="$up" \
+          'BEGIN{print (r2-r1) - (u2-u1)}')"
+
+        abs="$(awk -v d="$drift" 'BEGIN{print (d<0)?-d:d}')"
+
+        if awk -v a="$abs" -v t="$threshold" 'BEGIN{exit !(a>=t)}'; then
+          systemctl start ghaf-journal-alloy-recover.service || true
+        fi
+
+        last_real="$real"
+        last_up="$up"
+      done
+    '';
+  };
+
+  ghafJournalAlloyRecover = pkgs.writeShellApplication {
+    name = "ghaf-journal-alloy-recover";
+    runtimeInputs = with pkgs; [
+      coreutils
+      systemd
+    ];
+    text = ''
+      stamp="/run/ghaf-journal-alloy-recover.stamp"
+      now="$(date +%s)"
+      cooldown="${toString recCfg.cooldownSeconds}"
+
+      if [ -e "$stamp" ]; then
+        last="$(cat "$stamp" 2>/dev/null || echo 0)"
+        if [ "$((now-last))" -lt "$cooldown" ]; then
+          exit 0
+        fi
+      fi
+      echo "$now" > "$stamp"
+
+      systemd-tmpfiles --create --prefix /var/log/journal
+      systemctl restart systemd-journald.service
+      systemctl restart alloy.service
+    '';
+  };
 in
 {
   # Creating logging configuration options needed across the host and vms
@@ -78,5 +143,60 @@ in
         default = "1day";
       };
     };
+
+    recovery = {
+      enable = mkOption {
+        description = "Recover journald/alloy after a realtime clock jump (e.g., manual clock change).";
+        type = types.bool;
+        default = false;
+      };
+
+      thresholdSeconds = mkOption {
+        description = "Only act on clock jumps >= this many seconds.";
+        type = types.int;
+        default = 30;
+      };
+
+      intervalSeconds = mkOption {
+        description = "Polling interval used by the clock-jump watcher.";
+        type = types.int;
+        default = 5;
+      };
+
+      cooldownSeconds = mkOption {
+        description = "Minimum time between recover executions.";
+        type = types.int;
+        default = 60;
+      };
+    };
+  };
+
+  config = mkIf (config.ghaf.logging.enable && recCfg.enable) {
+
+    # Watcher: detects realtime jumps by comparing realtime vs monotonic progression
+    systemd.services.ghaf-clock-jump-watcher = {
+      description = "Detect realtime clock jumps and trigger journald/alloy recovery";
+      wantedBy = [ "multi-user.target" ];
+
+      serviceConfig = {
+        Type = "simple";
+        Restart = "always";
+        RestartSec = 2;
+        ExecStart = lib.getExe ghafClockJumpWatcher;
+      };
+    };
+
+    systemd.services.ghaf-journal-alloy-recover = {
+      description = "Recover journald/alloy after time jump";
+
+      unitConfig = {
+        StartLimitIntervalSec = "0";
+      };
+
+      serviceConfig = {
+        Type = "oneshot";
+        ExecStart = lib.getExe ghafJournalAlloyRecover;
+      };
+    };
   };
 }

modules/common/logging/server.nix

@@ -177,12 +177,15 @@ in
           stage.drop {
             expression = "(GatewayAuthenticator::login|Gateway login succeeded|csd-wrapper|nmcli)"
           }
+          stage.drop {
+            older_than = "168h"
+          }
         }
 
         loki.source.journal "journal" {
           path          = "/var/log/journal"
           relabel_rules = discovery.relabel.adminJournal.rules
-          forward_to    = [loki.write.remote.receiver]
+          forward_to    = [loki.process.system.receiver]
         }
 
         loki.write "remote" {
@@ -205,7 +208,7 @@ in
           // system in order to guarantee persistence of acknowledged data.
           wal {
             enabled = true
-            max_segment_age = "240h"
+            max_segment_age = "168h"
             drain_timeout = "4s"
           }
           external_labels = { machine = local.file.macAddress.content }
@@ -229,6 +232,9 @@ in
     services.alloy.enable = true;
 
     systemd.services.alloy.serviceConfig = {
+      after = [ "systemd-journald.service" ];
+      requires = [ "systemd-journald.service" ];
+
       # If there is no internet connection , shutdown/reboot will take around 100sec
       # So, to fix that problem we need to add stop timeout
       # https://github.com/grafana/loki/issues/6533

modules/microvm/sysvms/adminvm.nix

@@ -91,6 +91,7 @@ let
                   };
                 };
               };
+              recovery.enable = true;
             };
 
             security.fail2ban.enable = configHost.ghaf.development.ssh.daemon.enable;