Merge pull request #1 from NousResearch/2024-01-06-upstream-multiple-input

Mirror of upstream PR EleutherAI/lm-evaluation-harness#2576

Author: teknium1

lm_eval/api/task.py

@@ -450,6 +450,7 @@ def build_all_requests(
                 ctx=fewshot_ctx,
                 metadata=(self.config["task"], doc_id, self.config.repeats),
                 apply_chat_template=apply_chat_template,
+                chat_template=chat_template,
             )
 
             if not isinstance(inst, list):
@@ -1087,6 +1088,8 @@ def fewshot_context(
         example = self.doc_to_text(doc)
         if apply_chat_template:
             if self.multiple_input:
+                if not labeled_examples:
+                    return ""
                 return chat_template(labeled_examples)
             if isinstance(example, str):
                 self.append_target_question(
@@ -1303,6 +1306,7 @@ def construct_requests(
         self, doc: dict, ctx: str, **kwargs
     ) -> Union[List[Instance], Instance]:
         apply_chat_template = kwargs.pop("apply_chat_template", False)
+        chat_template: Callable | None = kwargs.pop("chat_template", None)
 
         aux_arguments = None
 
@@ -1317,9 +1321,20 @@ def construct_requests(
                 target_delimiter = ""
             if self.multiple_input:
                 # If there are multiple inputs, choices are placed in the ctx
+                # apply chat_template to choices if apply_chat_template
                 cont = self.doc_to_target(doc)
+
                 arguments = [
-                    (ctx + choice, f"{target_delimiter}{cont}") for choice in choices
+                    (
+                        ctx
+                        + (
+                            chat_template([{"role": "user", "content": choice}])
+                            if apply_chat_template
+                            else choice
+                        ),
+                        f"{target_delimiter}{cont}",
+                    )
+                    for choice in choices
                 ]
             else:
                 # Otherwise they are placed in the continuation