fixed mmlu generative response extraction

lm_eval/tasks/mmlu/generative/_default_template_yaml

@@ -14,6 +14,18 @@ metric_list:
   - metric: exact_match
     aggregation: mean
     higher_is_better: true
+filter_list:
+  - name: get_response
+    filter:
+      # Filter everything after the first break line
+      - function: "regex"
+        regex_pattern: "^(.*?)(?=\\n|$)"
+      # Remove leading white spaces
+      - function: remove_whitespace
+      # function to ignore right white spaces or line breaks
+      - function: "regex"
+        regex_pattern: "^(.*?)\\s*$"
+      - function: take_first
 metadata:
   version: 2.0
 dataset_kwargs:

lm_eval/tasks/mmlu/generative/_mmlu.yaml

@@ -1,33 +1,38 @@
-group: mmlu_generative
+group: mmlu_fix_generative
 group_alias: mmlu (generative)
 task:
   - group: stem
     task:
-      - mmlu_stem_generative
+      - mmlu_stem_fix_generative
     aggregate_metric_list:
-      - metric: acc
+      - metric: exact_match
         weight_by_size: True
+        filter_list: get_response
   - group: other
     task:
-      - mmlu_other_generative
+      - mmlu_other_fix_generative
     aggregate_metric_list:
-      - metric: acc
+      - metric: exact_match
         weight_by_size: True
+        filter_list: get_response
   - group: social sciences
     task:
-      - mmlu_social_sciences_generative
+      - mmlu_social_sciences_fix_generative
     aggregate_metric_list:
-      - metric: acc
+      - metric: exact_match
         weight_by_size: True
+        filter_list: get_response
   - group: humanities
     task:
-      - mmlu_humanities_generative
+      - mmlu_humanities_fix_generative
     aggregate_metric_list:
-      - metric: acc
+      - metric: exact_match
         weight_by_size: True
+        filter_list: get_response
 aggregate_metric_list:
   - aggregation: mean
     metric: exact_match
     weight_by_size: True
+    filter_list: get_response
 metadata:
   version: 2