x

Signed-off-by: SumanthRH <[email protected]>

Author: SumanthRH

skythought/evals/tasks/aime/aime25.yaml renamed to skythought/evals/tasks/aime/aime25_1.yaml

@@ -1,6 +1,7 @@
 handler: aime
 dataset_path: opencompass/AIME2025
-dataset_split: train
+dataset_subset: AIME2025-I
+dataset_split: test
 question_key: question
 answer_key: answer
 templating_parameters:

skythought/evals/tasks/aime/aime25_2.yaml

@@ -0,0 +1,9 @@
+handler: aime
+dataset_path: opencompass/AIME2025
+dataset_subset: AIME2025-II
+dataset_split: test
+question_key: question
+answer_key: answer
+templating_parameters:
+  template: "{prompt}\nReturn your final response within \\boxed{{}}"
+

skythought/evals/tasks/apps/apps_handler.py

@@ -36,14 +36,6 @@ def generate_prompt(self, problem):
     def check_correctness(self, problem, generation):
         TIMEOUT = 10
 
-        def _temp_run(problem, generation, debug, result):
-            try:
-                result.append(
-                    apps_run_test(problem=problem, test=generation, debug=debug)
-                )
-            except Exception:
-                pass
-
         manager = Manager()
         result = manager.list()
         p = multiprocessing.Process(
@@ -100,3 +92,10 @@ def load_and_filter_dataset(
         train_data = train_data.to_pandas()
 
         return train_data.iloc[start:end] if end > 0 else train_data.iloc[start:]
+
+
+def _temp_run(problem, generation, debug, result):
+    try:
+        result.append(apps_run_test(problem=problem, test=generation, debug=debug))
+    except Exception:
+        pass