add reasoning-gym verifier

pyproject.toml

@@ -3,7 +3,7 @@ name = "genesys"
 version = "0.1.0"
 description = "AGI"
 readme = "README.md"
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 dependencies = [
     "torch>=2.4.1",
     "datasets>=3.0.0",
@@ -16,6 +16,7 @@ dependencies = [
     "docker>=7.1.0",
     "pynvml>=12.0.0",
     "toploc>=0.0.2",
+    "reasoning-gym @ git+https://github.com/open-thought/reasoning-gym@72e45e9"
 ]
 
 [project.optional-dependencies]

src/genesys/verifiers/reasoning_gym_verifier.py

@@ -0,0 +1,30 @@
+from reasoning_gym.factory import DATASETS, get_score_answer_fn
+
+from genesys.schemas import Response
+from genesys.verifiers.base_verifier import BaseVerifier
+
+
+class ReasoningGymVerifier(BaseVerifier):
+    """
+    Verifier for procedural datasets from reasoning gym.
+    https://github.com/open-thought/reasoning-gym
+    """
+
+    def __init__(self):
+        del DATASETS["composite"]
+        self.score_answer_fns = {
+            dataset_name: get_score_answer_fn(name=dataset_name) for dataset_name in DATASETS.keys()
+        }
+
+    def verify(self, result: Response):
+        """
+        Evaluates the answer with the scoring function from the corresponding reasoning gym dataset.
+
+        The score is a float between 0 and 1.
+        """
+
+        dataset_name = result["source"]
+        entry = {"answer": result["verification_info"]["ground_truth"], "metadata": result["metadata"]}
+
+        score = self.score_answer_fns[dataset_name](answer=result["llm_response"], entry=entry)
+        return dict(score=score, verification_result_info={})

src/genesys/verifiers/registry.py

@@ -2,10 +2,12 @@
 from genesys.verifiers.math_verifier import MathVerifier
 from genesys.verifiers.llm_judge_verifier import LlmJudgeVerifier
 from genesys.verifiers.code_output_prediction_verifier import CodeUnderstandingVerifier
+from genesys.verifiers.reasoning_gym_verifier import ReasoningGymVerifier
 
 VERIFIER_REGISTRY = {
     "verifiable_code": CodeVerifier,
     "verifiable_math": MathVerifier,
     "llm_judgeable_groundtruth_similarity": LlmJudgeVerifier,
     "code_output_prediction": CodeUnderstandingVerifier,
+    "reasoning_gym": ReasoningGymVerifier,
 }