Add expected_output to the evaluation request

Author: juho-y

src/root_signals_mcp/client.py

@@ -120,7 +120,12 @@ async def list_evaluators(self) -> list[dict[str, Any]]:
         return result.get("evaluators", [])  # type: ignore
 
     async def run_evaluation(
-        self, evaluator_id: str, request: str, response: str, contexts: list[str] | None = None
+        self,
+        evaluator_id: str,
+        request: str,
+        response: str,
+        contexts: list[str] | None = None,
+        expected_output: str | None = None,
     ) -> dict[str, Any]:
         """Run a standard evaluation using a RootSignals evaluator by ID.
 
@@ -129,6 +134,7 @@ async def run_evaluation(
             request: The user request/query
             response: The model's response to evaluate
             contexts: Optional list of contexts (policy files, examples, etc.) used for generation. Only used for evaluators that require contexts.
+            expected_output: Optional expected LLM response. Only used for evaluators that require expected output.
 
         Returns:
             Evaluation result with score and justification
@@ -138,12 +144,18 @@ async def run_evaluation(
             "request": request,
             "response": response,
             "contexts": contexts,
+            "expected_output": expected_output,
         }
 
         return await self.call_tool("run_evaluation", arguments)
 
     async def run_evaluation_by_name(
-        self, evaluator_name: str, request: str, response: str, contexts: list[str] | None = None
+        self,
+        evaluator_name: str,
+        request: str,
+        response: str,
+        contexts: list[str] | None = None,
+        expected_output: str | None = None,
     ) -> dict[str, Any]:
         """Run a standard evaluation using a RootSignals evaluator by name.
 
@@ -152,6 +164,7 @@ async def run_evaluation_by_name(
             request: The user request/query
             response: The model's response to evaluate
             contexts: Optional list of contexts (policy files, examples, etc.) used for generation. Only used for evaluators that require contexts.
+            expected_output: Optional expected LLM response. Only used for evaluators that require expected output.
 
         Returns:
             Evaluation result with score and justification
@@ -161,6 +174,7 @@ async def run_evaluation_by_name(
             "request": request,
             "response": response,
             "contexts": contexts,
+            "expected_output": expected_output,
         }
 
         return await self.call_tool("run_evaluation_by_name", arguments)

src/root_signals_mcp/evaluator.py

@@ -116,6 +116,7 @@ async def run_evaluation(self, request: EvaluationRequest) -> EvaluationResponse
                 request=request.request,
                 response=request.response,
                 contexts=request.contexts,
+                expected_output=request.expected_output,
             )
 
             return result
@@ -147,6 +148,7 @@ async def run_evaluation_by_name(self, request: EvaluationRequestByName) -> Eval
                 request=request.request,
                 response=request.response,
                 contexts=request.contexts,
+                expected_output=request.expected_output,
             )
 
             return result

src/root_signals_mcp/schema.py

@@ -171,14 +171,6 @@ class RequiredInput(BaseModel):
     items: ArrayInputItem | None = None
 
 
-INPUTS_DESCRIPTION = """
-Schema defining the input parameters required for running the evaluator (run_evaluation parameters).
-If contexts are required, it means this is a RAG evaluator and you must pass contexts such as policy files, examples, etc.
-If expected_output is required, it means this is a gold standard output evaluator and you must pass the expected output.
-Request and response are required for almost all evaluators. Request is the user query and response is the model's response to evaluate.
-"""
-
-
 class EvaluatorInfo(BaseRootSignalsModel):
     """
     Model for evaluator information.
@@ -192,7 +184,7 @@ class EvaluatorInfo(BaseRootSignalsModel):
     intent: str | None = Field(None, description="Intent of the evaluator")
     inputs: dict[str, RequiredInput] = Field(
         ...,
-        description=INPUTS_DESCRIPTION,
+        description="Schema defining the input parameters required for running the evaluator (run_evaluation parameters).",
     )
 
     @property

src/root_signals_mcp/test/test_evaluator.py

@@ -161,6 +161,7 @@ async def test_run_evaluation_passes_correct_parameters(mock_api_client: MagicMo
         request="Test request",
         response="Test response",
         contexts=["Test context"],
+        expected_output="Test expected output",
     )
 
     result = await service.run_evaluation(request)
@@ -170,6 +171,7 @@ async def test_run_evaluation_passes_correct_parameters(mock_api_client: MagicMo
         request="Test request",
         response="Test response",
         contexts=["Test context"],
+        expected_output="Test expected output",
     )
 
     assert result.evaluator_name == "Test Evaluator"
@@ -195,6 +197,7 @@ async def test_run_evaluation_by_name_passes_correct_parameters(mock_api_client:
         request="Test request",
         response="Test response",
         contexts=["Test context"],
+        expected_output="Test expected output",
     )
 
     result = await service.run_evaluation_by_name(request)
@@ -204,6 +207,7 @@ async def test_run_evaluation_by_name_passes_correct_parameters(mock_api_client:
         request="Test request",
         response="Test response",
         contexts=["Test context"],
+        expected_output="Test expected output",
     )
 
     assert result.evaluator_name == "Test Evaluator"