feat: analytics retention

Author: enitrat

API_DOCUMENTATION.md

@@ -352,6 +352,112 @@ Common cases:
 - `404 Not Found` — unknown agent id.
 - `500 Internal Server Error` — unexpected backend issues.
 
+## Query Insights
+
+The Query Insights API exposes raw interaction logs and lightweight analytics for downstream processing.
+
+### `GET /v1/insights/queries`
+
+Fetch paginated user queries within a specific window.
+
+- `start_date` _(ISO 8601, required)_ — inclusive lower bound.
+- `end_date` _(ISO 8601, required)_ — inclusive upper bound.
+- `agent_id` _(optional)_ — filter by agent id when provided.
+- `limit` _(default `100`)_ — maximum rows returned.
+- `offset` _(default `0`)_ — pagination offset.
+
+**Response** `200 OK`
+
+```json
+{
+  "items": [
+    {
+      "id": "ad0c2b34-04ab-4d0a-9855-47c19f0f2830",
+      "created_at": "2024-04-01T12:30:45.123456+00:00",
+      "agent_id": "cairo-coder",
+      "final_user_query": "How do I declare a storage variable in Cairo 1?"
+    }
+  ],
+  "total": 1,
+  "limit": 100,
+  "offset": 0
+}
+```
+
+### `POST /v1/insights/analyze`
+
+Trigger an asynchronous analysis job. The response returns immediately with the job identifier; the analysis runs in the background.
+
+#### Request
+
+```json
+{
+  "start_date": "2024-04-01T00:00:00Z",
+  "end_date": "2024-04-15T23:59:59Z",
+  "agent_id": "cairo-coder"
+}
+```
+
+**Response** `202 Accepted`
+
+```json
+{
+  "analysis_id": "88ed4a1e-1bda-45b9-a3e8-5c6df8b6f1f1",
+  "status": "pending"
+}
+```
+
+### `GET /v1/insights/analyses`
+
+List recent analysis jobs.
+
+**Response** `200 OK`
+
+```json
+[
+  {
+    "id": "88ed4a1e-1bda-45b9-a3e8-5c6df8b6f1f1",
+    "created_at": "2024-04-15T12:00:00+00:00",
+    "status": "completed",
+    "analysis_parameters": {
+      "start_date": "2024-04-01T00:00:00+00:00",
+      "end_date": "2024-04-15T23:59:59+00:00",
+      "agent_id": "cairo-coder"
+    }
+  }
+]
+```
+
+### `GET /v1/insights/analyses/{analysis_id}`
+
+Fetch a specific analysis job. If the job completed successfully, `analysis_result` contains the summarized metrics; otherwise `error_message` explains the failure.
+
+**Response** `200 OK`
+
+```json
+{
+  "id": "88ed4a1e-1bda-45b9-a3e8-5c6df8b6f1f1",
+  "created_at": "2024-04-15T12:00:00+00:00",
+  "status": "completed",
+  "analysis_parameters": {
+    "start_date": "2024-04-01T00:00:00+00:00",
+    "end_date": "2024-04-15T23:59:59+00:00",
+    "agent_id": "cairo-coder"
+  },
+  "analysis_result": {
+    "total_queries": 42,
+    "average_word_count": 18.6,
+    "top_terms": [
+      ["cairo", 7],
+      ["storage", 4]
+    ]
+  },
+  "error_message": null
+}
+```
+
+If the job id is unknown, the server responds with `404 Not Found`.
+
 ## Versioning & Compatibility
 
 - Current API version: `1.0.0` (see FastAPI metadata).

python/pyproject.toml

@@ -142,6 +142,9 @@ strict_optional = true
 testpaths = ["tests"]
 pythonpath = ["src"]
 asyncio_mode = "auto"
+markers = [
+  "db: marks tests that require a database (run by default, use -m 'not db' to skip)",
+]
 filterwarnings = [
   "ignore::DeprecationWarning",
   "ignore::PendingDeprecationWarning",

python/src/cairo_coder/core/rag_pipeline.py

@@ -82,6 +82,11 @@ def __init__(self, config: RagPipelineConfig):
         self._current_processed_query: ProcessedQuery | None = None
         self._current_documents: list[Document] = []
 
+    @property
+    def last_retrieved_documents(self) -> list[Document]:
+        """Documents retrieved during the most recent pipeline execution."""
+        return self._current_documents
+
     async def _aprocess_query_and_retrieve_docs(
         self,
         query: str,

python/src/cairo_coder/db/__init__.py

@@ -0,0 +1,31 @@
+"""
+Database utilities for the Cairo Coder server.
+
+This package exposes helpers for initializing the asyncpg connection pool and
+provides Pydantic representations used when persisting query insights data.
+"""
+
+from .models import QueryAnalysis, UserInteraction
+from .repository import (
+    create_analysis_job,
+    create_user_interaction,
+    get_analysis_job_by_id,
+    get_analysis_jobs,
+    get_interactions,
+    update_analysis_job,
+)
+from .session import close_pool, execute_schema_scripts, get_pool
+
+__all__ = [
+    "QueryAnalysis",
+    "UserInteraction",
+    "create_analysis_job",
+    "create_user_interaction",
+    "get_analysis_job_by_id",
+    "get_analysis_jobs",
+    "get_interactions",
+    "update_analysis_job",
+    "close_pool",
+    "execute_schema_scripts",
+    "get_pool",
+]

python/src/cairo_coder/db/models.py

@@ -0,0 +1,36 @@
+"""
+Pydantic models representing rows stored in the query insights database tables.
+"""
+
+from __future__ import annotations
+
+import uuid
+from datetime import datetime, timezone
+from typing import Any, Optional
+
+from pydantic import BaseModel, Field
+
+
+class UserInteraction(BaseModel):
+    """Represents a record in the user_interactions table."""
+
+    id: uuid.UUID = Field(default_factory=uuid.uuid4)
+    created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    agent_id: str
+    mcp_mode: bool = False
+    chat_history: Optional[list[dict[str, Any]]] = None
+    query: str
+    generated_answer: Optional[str] = None
+    retrieved_sources: Optional[list[dict[str, Any]]] = None
+    llm_usage: Optional[dict[str, Any]] = None
+
+
+class QueryAnalysis(BaseModel):
+    """Represents a record in the query_analyses table."""
+
+    id: uuid.UUID = Field(default_factory=uuid.uuid4)
+    created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    status: str = "pending"
+    analysis_parameters: dict[str, Any]
+    analysis_result: Optional[dict[str, Any]] = None
+    error_message: Optional[str] = None