Introduce new scoring APIs for curation + training

.pre-commit-config.yaml

@@ -5,12 +5,12 @@ repos:
       - id: ruff
         args: [ --fix, --exit-non-zero-on-fix ]
         # NOTE (sumanthrh): Many of the files excluded here are used for validating code generation, and linters do not recognize some of the logic in these files. skythought/train is excluded for now because it's a fork of Llamafactory
-        exclude: (^skythought/train/.*|^skythought/skythought-rl/.*|tasks/taco/pyext2\.py|tasks/taco/taco_util\.py|tasks/apps/apps_util\.py|scripts/prompts\.py|skythought/test-time-scaling/.*)$
+        exclude: (^skythought/train/.*|^skythought/skythought-rl/.*|pyext2\.py|taco_util\.py|apps_util\.py|scripts/prompts\.py|skythought/test-time-scaling/.*)$
 
 
   # Black needs to be ran after ruff with --fix
   - repo: https://github.com/psf/black
     rev: 24.10.0
     hooks:
       - id: black
-        exclude: (^skythought/train/.*|^skythought/skythought-rl/.*|tasks/taco/pyext2\.py|skythought/test-time-scaling/.*)$
+        exclude: (^skythought/train/.*|^skythought/skythought-rl/.*|pyext2\.py|skythought/test-time-scaling/.*)$

pyproject.toml

@@ -8,7 +8,6 @@ authors = [
 requires-python = ">=3.9,<3.11"
 dependencies = [
     "vllm==0.7.0",
-    "pyext",
     "word2number",
     "scipy",
     "datasets",
@@ -44,7 +43,7 @@ skythought = ["evals/**/*.yaml", "evals/**/*.yml"]
 skythought = "skythought.evals.cli:main"
 
 [project.optional-dependencies]
-dev = ["pytest", "pytest-mock", "black", "ruff", "pre-commit"]
+dev = ["pytest", "pytest-mock", "pytest-asyncio", "black", "ruff", "pre-commit"]
 
 [tool.ruff]
 line-length = 160

recipes/sky-t1-preview/postprocess.py

@@ -0,0 +1,44 @@
+from typing import Any, Dict
+
+STILL2_SYSTEM_PROMPT = "Your role as an assistant involves thoroughly exploring questions through a systematic long \
+thinking process before providing the final precise and accurate solutions. This requires \
+engaging in a comprehensive cycle of analysis, summarizing, exploration, reassessment, reflection, \
+backtracing, and iteration to develop well-considered thinking process. \
+Please structure your response into two main sections: Thought and Solution. \
+In the Thought section, detail your reasoning process using the specified format: \
+<|begin_of_thought|> {thought with steps separated with '\n\n'} \
+<|end_of_thought|> \
+Each step should include detailed considerations such as analisying questions, summarizing \
+relevant findings, brainstorming new ideas, verifying the accuracy of the current steps, refining \
+any errors, and revisiting previous steps. \
+In the Solution section, based on various attempts, explorations, and reflections from the Thought \
+section, systematically present the final solution that you deem correct. The solution should \
+remain a logical, accurate, concise expression style and detail necessary step needed to reach the \
+conclusion, formatted as follows: \
+<|begin_of_solution|> \
+{final formatted, precise, and clear solution} \
+<|end_of_solution|> \
+Now, try to solve the following question through the above guidelines:"
+
+
+def convert_to_sharegpt_format(row: Dict[str, Any], prompt_column, response_column):
+    prompt = row[prompt_column]
+    # accept
+    # Create the conversation format
+    conversations = [
+        {"from": "user", "value": prompt},
+        {
+            "from": "assistant",
+            "value": row[response_column],
+        },
+    ]
+
+    # Prepare the final structure
+    cur_data = {
+        "system": STILL2_SYSTEM_PROMPT,
+        "conversations": conversations,
+        # TODO: remove this
+        **row,
+    }
+
+    return cur_data

recipes/sky-t1-preview/preprocess.py

@@ -0,0 +1,87 @@
+import json
+
+import pyarrow as pa
+from ray.data import Schema
+
+
+class APPSPreprocessor:
+    WITH_FN_NAME_TEMPLATE = "Generate an executable Python function generated from the given prompt. The function should take stdin as input and print the output. Simply call the function after the definition. {prompt}"  # noqa: E501
+
+    WITHOUT_FN_NAME_TEMPLATE = "Generate an executable Python function generated from the given prompt. Return the function body without invoking it at the final solution. {prompt}"  # noqa: E501
+
+    WITH_STARTER_CODE_TEMPLATE = "{input}\n{starter_code}"
+
+    def __call__(self, row):
+        test_case = json.loads(row["input_output"])
+        starter_code = row["starter_code"]
+        prompt = row["question"]
+        if not test_case.get("fn_name"):
+            _input = self.WITH_FN_NAME_TEMPLATE.format(prompt=prompt)
+        else:
+            _input = self.WITHOUT_FN_NAME_TEMPLATE.format(prompt=prompt)
+
+        if starter_code is not None:
+            _input = self.WITH_STARTER_CODE_TEMPLATE.format(
+                input=_input, starter_code=starter_code
+            )
+
+        return {**row, "user_input": _input}
+
+
+class TACOPreprocessor:
+    INITIAL_TEMPLATE = "\nQUESTION:\n{prompt}"
+    STARTER_CODE_TEMPLATE = "{input}\n{starter_code}"
+    STDIN_TEMPLATE = "{input}\nUse Standard Input format\nANSWER:\n"
+    CALL_TEMPLATE = "{input}\nUse Call-Based format\nANSWER:\n"
+
+    def __call__(self, problem):
+
+        prompt = problem["question"]
+        starter_code = (
+            None if len(problem["starter_code"]) == 0 else problem["starter_code"]
+        )
+        try:
+            input_outpout = json.loads(problem["input_output"])
+            fn_name = (
+                None if not input_outpout.get("fn_name") else input_outpout["fn_name"]
+            )
+        except ValueError:
+            fn_name = None
+
+        _input = self.INITIAL_TEMPLATE.format(prompt=prompt)
+
+        if starter_code:
+            _input = self.STARTER_CODE_TEMPLATE.format(
+                input=_input, starter_code=starter_code
+            )
+        else:
+            _input = self.INITIAL_TEMPLATE.format(prompt=prompt)
+        if (not fn_name) and (not starter_code):
+            _input = self.STDIN_TEMPLATE.format(input=_input)
+        else:
+            _input = self.CALL_TEMPLATE.format(input=_input)
+
+        return {**problem, "user_input": _input}
+
+
+class NUMINAPreprocessor:
+    TEMPLATE = "Return your final response within \\boxed{{}}. {prompt}"
+
+    def __call__(self, row):
+        prompt = row["problem"]
+        _input = self.TEMPLATE.format(prompt=prompt)
+        return {**row, "user_input": _input}
+
+
+def taco_coerce_types(row, schema: Schema):
+    for key, schema_type in zip(schema.names, schema.types):
+        value = pa.array([row[key]])
+        if value.type != schema_type:
+            if schema_type == pa.string():
+                try:
+                    row[key] = str(row[key])
+                except Exception:
+                    row[key] = ""
+            elif schema_type == pa.null():
+                row[key] = None
+    return row

recipes/sky-t1-preview/prompts.py

@@ -0,0 +1,69 @@
+CONVERT_PROMPT_EXAMPLE = (  # noqa: E501
+    "<|begin_of_thought|>\n\n"
+    "Okay, so I've got this problem here. Mr. Wang leaves home at 6 AM, riding his bike at 12 km/h, "
+    "and he stops to rest for 6 minutes after every 30 minutes of riding. Then, when he arrives at a park "
+    "that's 16.8 km away, I need to find out the angle between the hour and minute hands on his watch.\n\n"
+    "Alright, first things first, I need to figure out how long it takes Mr. Wang to ride 16.8 km, including "
+    "his rest periods.\n\n"
+    "So, his speed is 12 km/h. To find out how long it takes to go 16.8 km without any stops, I can use the formula "
+    "time = distance/speed. That would be 16.8 divided by 12, which is 1.4 hours. To make it easier, that's 1 hour and 24 minutes.\n\n"
+    "But wait, he doesn't ride straight through. He stops for 6 minutes after every 30 minutes of riding. So, I need to see how many "
+    "of those 30-minute riding periods are there in his total riding time.\n\n"
+    "In 1 hour and 24 minutes of riding, how many 30-minute segments are there? Well, 1 hour is 60 minutes, plus 24 minutes makes 84 minutes "
+    "total riding time. So, 84 divided by 30 is 2.8. That means he has two full 30-minute riding periods and a partial one.\n\n"
+    "After each full 30-minute riding period, he rests for 6 minutes. So, for two full periods, he rests twice, which is 12 minutes of rest.\n\n"
+    "Now, for the partial riding period. Since 2 times 30 minutes is 60 minutes, and he has 84 minutes of riding, the remaining riding time is 84 minus 60, "
+    "which is 24 minutes. So, he rides for 24 minutes without another rest because he doesn't complete another 30-minute segment.\n\n"
+    "So, total time taken is riding time plus rest time. That's 84 minutes riding plus 12 minutes resting, totaling 96 minutes.\n\n"
+    "Wait a minute, but he stops after every 30 minutes of riding, but in the last partial period of 24 minutes, does he rest again? I think he only rests after "  # noqa: E501
+    "completing 30 minutes of riding, so in this case, since the last riding period is only 24 minutes, he doesn't take an additional rest after that.\n\n"
+    "So, total time should be 84 minutes riding plus 12 minutes resting, which is indeed 96 minutes, or 1 hour and 36 minutes.\n\n"
+    "So, he leaves at 6 AM and takes 1 hour and 36 minutes to reach the park, arriving at 7:36 AM.\n\n"
+    "Now, I need to find the angle between the hour and minute hands at 7:36.\n\n"
+    "To find the angle between the hour and minute hands, I can use the formula:\n\n"
+    "|30H - 5.5M|\n\n"
+    "where H is the hour and M is the minutes.\n\n"
+    "At 7:36, H is 7 and M is 36.\n\n"
+    "So, plugging in:\n\n"
+    "30*7 = 210\n\n"
+    "5.5*36 = 198\n\n"
+    "210 - 198 = 12\n\n"
+    "So, the angle is 12 degrees.\n\n"
+    "Wait, but I should make sure that's the smaller angle. Sometimes, the larger angle is considered, but usually, the smaller one is what is asked for.\n\n"
+    "So, the angle between the hour and minute hands at 7:36 AM is 12 degrees.\n\n"
+    "I think that's the answer.<|end_of_thought|>\n\n"
+    "<|begin_of_solution|>\n\n"
+    "Mr. Wang leaves home at 6 AM and rides at a speed of 12 km/h, stopping to rest for 6 minutes after every 30 minutes of riding. "
+    "He arrives at a park 16.8 km away. To determine the angle between the hour and minute hands on his watch when he arrives, we first calculate the total time taken.\n\n"  # noqa: E501
+    "1. **Riding time without stops**:\n\n"
+    "$$\\text{Time} = \\frac{\\text{Distance}}{\\text{Speed}} = \\frac{16.8 \\text{ km}}{12 \\text{ km/h}} = 1.4 \\text{ hours} = 84 \\text{ minutes}$$\n\n"
+    "2. **Rest periods**:\n\n"
+    "  - He rests for 6 minutes after every 30 minutes of riding.\n\n"
+    "  - In 84 minutes of riding, he completes 2 full 30-minute segments and a partial 24-minute segment.\n\n"
+    "  - He rests twice, totaling 12 minutes of rest.\n\n"
+    "3. **Total time**:\n\n"
+    "$$\\text{Total time} = 84 \\text{ minutes (riding)} + 12 \\text{ minutes (rest)} = 96 \\text{ minutes} = 1 \\text{ hour and } 36 \\text{ minutes}$$\n\n"
+    "  - He arrives at 7:36 AM.\n\n"
+    "4. **Angle between hour and minute hands at 7:36**:\n\n"
+    "  - Use the formula:\n\n"
+    "$$\\text{Angle} = |30H - 5.5M|$$\n\n"
+    "  - At 7:36, $H = 7$ and $M = 36$:\n\n"
+    "$$\\text{Angle} = |30 \\times 7 - 5.5 \\times 36| = |210 - 198| = 12 \\text{ degrees}$$\n\n"
+    "Thus, the angle between the hour and minute hands on his watch is $\\boxed{12}$.<|end_of_solution|>\n"  # noqa: E501
+)
+
+# From https://arxiv.org/pdf/2412.09413
+CONVERT_PROMPT = (
+    "Another solution is written in an unstructured way. Your job is to convert them into two sections:"
+    "<|begin_of_thought|>"
+    "(Thought process, you should copy exactly the thinking process of the original solution.)"
+    "<|end_of_thought|>"
+    "<|begin_of_solution|>"
+    "(Final formatted, precise, and clear solution; make sure there is only one solution in this section; If it is a coding problem, make sure there is only one code block)"  # noqa: E501
+    "<|end_of_solution|>"
+    "Here is an example demonstration of a different question, you can refer to its format: "
+    "{example}\n"
+    "Important: You should almost copy all the contents word-by-word of the original solution. Just convert them into two sections. "
+    "Make sure you include: <|begin_of_slow_thought|>, <|end_of_slow_thought|>,  <|begin_of_solution|>,<|end_of_solution|>  These four headers explicitly. "
+    "Content to be converted: {content}"
+)