Add expected_output to the evaluation request

Author: juho-y

src/root_signals_mcp/client.py

@@ -120,7 +120,12 @@ async def list_evaluators(self) -> list[dict[str, Any]]:
         return result.get("evaluators", [])  # type: ignore
 
     async def run_evaluation(
-        self, evaluator_id: str, request: str, response: str, contexts: list[str] | None = None
+        self,
+        evaluator_id: str,
+        request: str,
+        response: str,
+        contexts: list[str] | None = None,
+        expected_output: str | None = None,
     ) -> dict[str, Any]:
         """Run a standard evaluation using a RootSignals evaluator by ID.
 
@@ -129,6 +134,7 @@ async def run_evaluation(
             request: The user request/query
             response: The model's response to evaluate
             contexts: Optional list of contexts (policy files, examples, etc.) used for generation. Only used for evaluators that require contexts.
+            expected_output: Optional expected LLM response. Only used for evaluators that require expected output.
 
         Returns:
             Evaluation result with score and justification
@@ -138,12 +144,18 @@ async def run_evaluation(
             "request": request,
             "response": response,
             "contexts": contexts,
+            "expected_output": expected_output,
         }
 
         return await self.call_tool("run_evaluation", arguments)
 
     async def run_evaluation_by_name(
-        self, evaluator_name: str, request: str, response: str, contexts: list[str] | None = None
+        self,
+        evaluator_name: str,
+        request: str,
+        response: str,
+        contexts: list[str] | None = None,
+        expected_output: str | None = None,
     ) -> dict[str, Any]:
         """Run a standard evaluation using a RootSignals evaluator by name.
 
@@ -152,6 +164,7 @@ async def run_evaluation_by_name(
             request: The user request/query
             response: The model's response to evaluate
             contexts: Optional list of contexts (policy files, examples, etc.) used for generation. Only used for evaluators that require contexts.
+            expected_output: Optional expected LLM response. Only used for evaluators that require expected output.
 
         Returns:
             Evaluation result with score and justification
@@ -161,6 +174,7 @@ async def run_evaluation_by_name(
             "request": request,
             "response": response,
             "contexts": contexts,
+            "expected_output": expected_output,
         }
 
         return await self.call_tool("run_evaluation_by_name", arguments)

src/root_signals_mcp/schema.py

@@ -73,7 +73,11 @@ class BaseEvaluationRequest(BaseRootSignalsModel):
     response: str = Field(..., description="The AI assistant's response to evaluate")
     contexts: list[str] = Field(
         default=[],
-        description="List of required context strings for evaluation. This is only used for RAG evaluators that require contexts to be sent",
+        description="List of required context strings for evaluation. Used only for evaluators that have 'contexts' defined in their inputs.",
+    )
+    expected_output: str | None = Field(
+        default=None,
+        description="The expected LLM response. Used only for evaluators that have 'expected_output' defined in their inputs.",
     )
 
     @field_validator("request", "response")
@@ -167,14 +171,6 @@ class RequiredInput(BaseModel):
     items: ArrayInputItem | None = None
 
 
-INPUTS_DESCRIPTION = """
-Schema defining the input parameters required for running the evaluator (run_evaluation parameters).
-If 'contexts' is defined, it means this is a RAG evaluator and contexts should include policy files, examples, etc.
-If 'expected_output' is defined, it means this is a gold standard output evaluator and the expected output should be passed.
-Most evaluators require both request (the user query) and response (the model's response to evaluate) without them being explicitly defined.
-"""
-
-
 class EvaluatorInfo(BaseRootSignalsModel):
     """
     Model for evaluator information.
@@ -188,7 +184,7 @@ class EvaluatorInfo(BaseRootSignalsModel):
     intent: str | None = Field(None, description="Intent of the evaluator")
     inputs: dict[str, RequiredInput] = Field(
         ...,
-        description=INPUTS_DESCRIPTION,
+        description="Schema defining the input parameters required for running the evaluator (run_evaluation parameters).",
     )