Bump version to v0.4.4 ; Fixes to TMMLUplus (#2280)

Author: haileyschoelkopf

lm_eval/__main__.py

@@ -299,13 +299,6 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:
             "When `fewshot_as_multiturn` is selected, `apply_chat_template` must be set (either to `True` or to the chosen template name)."
         )
 
-    if (
-        args.num_fewshot is None or args.num_fewshot == 0
-    ) and args.fewshot_as_multiturn:
-        raise ValueError(
-            "If fewshot_as_multiturn is set, num_fewshot must be greater than 0."
-        )
-
     if args.include_path is not None:
         eval_logger.info(f"Including path: {args.include_path}")
     task_manager = TaskManager(args.verbosity, include_path=args.include_path)

lm_eval/tasks/__init__.py

@@ -489,10 +489,12 @@ def _get_task_and_group(self, task_dir: str):
                             if attr in config:
                                 if attr == "group" and print_info:
                                     self.logger.info(
-                                        "`group` and `group_alias` keys in tasks' configs will no longer be used in the next release of lm-eval. "
-                                        "`tag` will be used to allow to call a collection of tasks just like `group`. "
-                                        "`group` will be removed in order to not cause confusion with the new ConfigurableGroup "
-                                        "which will be the official way to create groups with addition of group-wide configurations."
+                                        "`group` and `group_alias` keys in TaskConfigs are deprecated and will be removed in v0.4.5 of lm_eval. "
+                                        "The new `tag` field will be used to allow for a shortcut to a group of tasks one does not wish to aggregate metrics across. "
+                                        "`group`s which aggregate across subtasks must be only defined in a separate group config file, "
+                                        "which will be the official way to create groups that support cross-task aggregation as in `mmlu`. "
+                                        "Please see the v0.4.4 patch notes and our documentation: https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/new_task_guide.md#advanced-group-configs "
+                                        "for more information."
                                     )
                                     print_info = False
                                     # attr = "tag"

lm_eval/tasks/tmmluplus/default/_tmmluplus.yaml

@@ -0,0 +1,13 @@
+group: tmmluplus
+task:
+- tmmluplus_other
+- tmmluplus_social_sciences
+- tmmluplus_humanities
+- tmmluplus_STEM
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+  - metric: acc_norm
+    weight_by_size: True
+metadata:
+  version: 2.0

lm_eval/tasks/tmmluplus/default/_tmmluplus_STEM.yaml

@@ -0,0 +1,10 @@
+group: tmmluplus_STEM
+task:
+- tmmluplus_STEM_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+  - metric: acc_norm
+    weight_by_size: True
+metadata:
+  version: 2.0

lm_eval/tasks/tmmluplus/default/_tmmluplus_humanities.yaml

@@ -0,0 +1,10 @@
+group: tmmluplus_humanities
+task:
+- tmmluplus_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+  - metric: acc_norm
+    weight_by_size: True
+metadata:
+  version: 2.0

lm_eval/tasks/tmmluplus/default/_tmmluplus_other.yaml

@@ -0,0 +1,10 @@
+group: tmmluplus_other
+task:
+- tmmluplus_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+  - metric: acc_norm
+    weight_by_size: True
+metadata:
+  version: 2.0

lm_eval/tasks/tmmluplus/default/_tmmluplus_social_sciences.yaml

@@ -0,0 +1,10 @@
+group: tmmluplus_social_sciences
+task:
+- tmmluplus_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+  - metric: acc_norm
+    weight_by_size: True
+metadata:
+  version: 2.0

lm_eval/tasks/tmmluplus/default/_default_template_yaml renamed to lm_eval/tasks/tmmluplus/default/_tmmluplus_template_yaml

@@ -16,4 +16,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  version: 1.0
+  version: 2.0

lm_eval/tasks/tmmluplus/default/tmmluplus.yaml

@@ -1,7 +1,6 @@
 "dataset_name": "accounting"
 "description": "以下為會計學的單選題，請提供正確答案的選項。\n\n"
-"group": "tmmluplus_other"
-"group_alias": "other"
-"include": "_default_template_yaml"
+"tag": "tmmluplus_other_tasks"
+"include": "_tmmluplus_template_yaml"
 "task": "tmmluplus_accounting"
 "task_alias": "accounting"