feat: implement per-module logging for worker pool

Add structured logging for individual module processing with detailed
tracking across all pipeline phases (clone, enrich, analyze).

Changes:
- Create module-logger.js utility for per-module log files
- Integrate logging into processModule() and worker processes
- Organize logs by run timestamp: logs/{runId}/modules/{moduleId}.log
- Auto-flush on errors and buffer overflow
- Add worker-specific ESLint overrides to central config
- Update worker pool README with logging documentation

Benefits:
- Isolated debugging per module (no log mixing in parallel execution)
- Complete audit trail with timestamps and structured data
- Worker PID tracking for troubleshooting
- Historical run comparison via timestamped directories

Author: KristjanESPERANTO

.gitignore

@@ -1,4 +1,5 @@
 /.pipeline-runs/
+/logs/
 /modules/*
 /modules_/*
 /modules_temp/*

docs/pipeline-refactor-roadmap.md

@@ -72,12 +72,20 @@ Merge stages 3+4+5 into parallel worker processes. See [worker-pool-design.md](p
 | P7.1 | ✅ Design complete           |
 | P7.2 | ✅ Single-worker prototype   |
 | P7.3 | ✅ Worker pool orchestration |
-| P7.4 | Per-module logging           |
+| P7.4 | ✅ Per-module logging        |
 | P7.5 | Cleanup old stage scripts    |
 | P7.6 | Incremental mode integration |
 
 **Note:** P7.6 (Incremental mode) deferred until after P7.5 (Cleanup). The existing cache logic in `scripts/check-modules/index.ts` continues to work; integrating it into the new worker architecture makes more sense once the old pipeline is removed.
 
+**P7.4 Implementation (Feb 2026):**
+
+- Created module-specific logger utility (`module-logger.js`)
+- Per-module log files organized by run timestamp: `logs/{runId}/modules/{moduleId}.log`
+- Detailed logging across all processing phases (clone, enrich, analyze)
+- Automatic buffer flushing on errors and completion
+- Integrated with existing worker pool architecture
+
 **P7.3 Implementation (Jan 2026):**
 
 - Worker process wrapper with IPC communication

eslint.config.js

@@ -102,6 +102,14 @@ export default defineConfig([
       "@typescript-eslint/no-explicit-any": "warn"
     }
   },
+  {
+    files: ["pipeline/workers/**/*.js"],
+    rules: {
+      "max-lines": ["warn", 900],
+      "max-lines-per-function": ["warn", 200],
+      "max-depth": ["warn", 6]
+    }
+  },
   { files: ["**/*.json"], ignores: ["package.json", "package-lock.json"], plugins: { json }, extends: ["json/recommended"], language: "json/json" },
   { files: ["package.json"], plugins: { packageJson }, extends: ["packageJson/recommended"], rules: { "package-json/sort-collections": "off" } },
   { files: ["**/*.md"], plugins: { markdown }, language: "markdown/gfm", extends: ["markdown/recommended"] }

pipeline/workers/README.md

@@ -83,9 +83,9 @@ Successfully tested with multiple module sets:
 
 ### Next Steps (P7.4+)
 
-- [ ] P7.4: Integrate incremental mode with module cache
-- [ ] P7.5: Add per-module logging to files
-- [ ] P7.6: Remove old stage scripts after migration complete
+- [x] P7.4: Per-module logging to files ✅
+- [ ] P7.5: Remove old stage scripts after migration complete
+- [ ] P7.6: Integrate incremental mode with module cache
 - [ ] P7.7: Performance benchmarking and optimization
 
 ### Configuration Options
@@ -148,6 +148,41 @@ See [../docs/pipeline/worker-pool-design.md](../../docs/pipeline/worker-pool-des
 
 ## Design Decisions
 
+### Per-Module Logging (P7.4) ✅
+
+Each module gets its own log file with detailed processing information:
+
+**Log Structure:**
+
+```text
+logs/
+  {runId}/              # e.g., 2026-02-04T10-30-45
+    modules/
+      MMM-Module-----Author.worker-12345.log
+      MMM-OtherModule-----Dev.worker-12346.log
+```
+
+**Features:**
+
+- Organized by run timestamp for historical tracking
+- Includes worker PID in filename for debugging
+- Structured log entries with phase, level, message, and optional data
+- Auto-flush on errors and when buffer reaches 100 entries
+- Closed automatically when module processing completes
+
+**Log Format:**
+
+```text
+[2026-02-04T10:30:45.123Z] [INFO] [clone] Starting clone stage {"url":"...","branch":"master"}
+[2026-02-04T10:30:47.456Z] [INFO] [clone] Repository cloned successfully
+[2026-02-04T10:30:47.500Z] [INFO] [enrich] Starting enrichment stage
+[2026-02-04T10:30:48.100Z] [INFO] [end] Module processing completed successfully {"processingTimeMs":2977}
+```
+
+**Usage:**
+
+The logger is automatically created for each module and passed via config. No manual setup required in module processing code.
+
 ### Single Module Processing Function
 
 Instead of calling separate stage scripts, `processModule()` executes all stages inline:

pipeline/workers/module-logger.js

@@ -0,0 +1,182 @@
+/**
+ * Per-Module Logging Utility (P7.4)
+ *
+ * Provides structured logging for individual module processing.
+ * Logs are written to files organized by run timestamp and module ID.
+ */
+
+import { ensureDirectory } from "../../scripts/shared/fs-utils.js";
+import fs from "node:fs/promises";
+import path from "node:path";
+
+/**
+ * @typedef {Object} ModuleLoggerOptions
+ * @property {string} projectRoot - Project root directory
+ * @property {string} runId - Unique run identifier (timestamp)
+ * @property {string} moduleId - Module identifier (name-----maintainer)
+ * @property {number} [workerId] - Worker process ID
+ */
+
+/**
+ * @typedef {Object} LogEntry
+ * @property {string} timestamp - ISO timestamp
+ * @property {string} level - Log level (info, warn, error, debug)
+ * @property {string} phase - Processing phase (clone, enrich, analyze)
+ * @property {string} message - Log message
+ * @property {Object} [data] - Additional structured data
+ */
+
+/**
+ * Create a module-specific logger that writes to file
+ *
+ * @param {ModuleLoggerOptions} options
+ * @returns {Promise<ModuleLogger>}
+ */
+export async function createModuleLogger(options) {
+  const { projectRoot, runId, moduleId, workerId } = options;
+
+  // Create logs directory structure: logs/{runId}/modules/
+  const logsDir = path.join(projectRoot, "logs", runId, "modules");
+  await ensureDirectory(logsDir);
+
+  // Sanitize module ID for filename (replace special chars)
+  const safeModuleId = moduleId.replace(/[^a-zA-Z0-9_-]/gu, "_");
+  const logFileName = workerId
+    ? `${safeModuleId}.worker-${workerId}.log`
+    : `${safeModuleId}.log`;
+
+  const logFilePath = path.join(logsDir, logFileName);
+
+  // Log buffer (written periodically and on close)
+  const logBuffer = [];
+  let closed = false;
+
+  /**
+   * Write buffered logs to file
+   */
+  async function flush() {
+    if (logBuffer.length === 0 || closed) {
+      return;
+    }
+
+    const content = `${logBuffer.join("\n")}\n`;
+    await fs.appendFile(logFilePath, content, "utf8");
+    logBuffer.length = 0;
+  }
+
+  /**
+   * Format log entry
+   * @param {LogEntry} entry
+   * @returns {string}
+   */
+  function formatLogEntry(entry) {
+    const { timestamp, level, phase, message, data } = entry;
+    const parts = [`[${timestamp}]`, `[${level.toUpperCase()}]`, `[${phase}]`, message];
+
+    if (data && Object.keys(data).length > 0) {
+      parts.push(JSON.stringify(data));
+    }
+
+    return parts.join(" ");
+  }
+
+  /**
+   * Add log entry
+   * @param {string} level
+   * @param {string} phase
+   * @param {string} message
+   * @param {Object} [data]
+   */
+  async function log(level, phase, message, data) {
+    if (closed) {
+      return;
+    }
+
+    const entry = {
+      timestamp: new Date().toISOString(),
+      level,
+      phase,
+      message,
+      ...(data && { data })
+    };
+
+    logBuffer.push(formatLogEntry(entry));
+
+    // Auto-flush on error or if buffer is large
+    if (level === "error" || logBuffer.length >= 100) {
+      await flush();
+    }
+  }
+
+  /**
+   * Close logger and flush remaining logs
+   */
+  async function close() {
+    if (closed) {
+      return;
+    }
+
+    await flush();
+    closed = true;
+  }
+
+  return {
+    /**
+     * Log info message
+     * @param {string} phase
+     * @param {string} message
+     * @param {Object} [data]
+     */
+    info: (phase, message, data) => log("info", phase, message, data),
+
+    /**
+     * Log warning message
+     * @param {string} phase
+     * @param {string} message
+     * @param {Object} [data]
+     */
+    warn: (phase, message, data) => log("warn", phase, message, data),
+
+    /**
+     * Log error message
+     * @param {string} phase
+     * @param {string} message
+     * @param {Object} [data]
+     */
+    error: (phase, message, data) => log("error", phase, message, data),
+
+    /**
+     * Log debug message
+     * @param {string} phase
+     * @param {string} message
+     * @param {Object} [data]
+     */
+    debug: (phase, message, data) => log("debug", phase, message, data),
+
+    /**
+     * Flush logs to file
+     */
+    flush,
+
+    /**
+     * Close logger and flush
+     */
+    close,
+
+    /**
+     * Get log file path
+     */
+    getLogFilePath: () => logFilePath
+  };
+}
+
+/**
+ * @typedef {Object} ModuleLogger
+ * @property {(phase: string, message: string, data?: Object) => Promise<void>} info
+ * @property {(phase: string, message: string, data?: Object) => Promise<void>} warn
+ * @property {(phase: string, message: string, data?: Object) => Promise<void>} error
+ * @property {(phase: string, message: string, data?: Object) => Promise<void>} debug
+ * @property {() => Promise<void>} flush
+ * @property {() => Promise<void>} close
+ * @property {() => string} getLogFilePath
+ */