fix: resolve duplicate task names and add safeguards. (#3394)

* fix: resolve duplicate task names and add safeguards.

This commit fixes 144 duplicate task names across MMLU-Redux and Flores translation benchmarks, and adds warnings and safeguards for more deterministic behavior.

Changes:
- Renamed all MMLU-Redux tasks from `mmlu_*_generative` to `mmlu_redux_*_generative` (57 tasks) to avoid conflicts with original  MMLU tasks that use different datasets (cais/mmlu vs edinburgh-dawg/mmlu-redux-2.0)

- Fixed Flores translation task duplicates by prefixing with benchmark name (e.g., `flores_ca-pt` → `catalan_bench_flores_ca-pt`). Updated generation scripts and regenerated YAMLs for catalan_bench, portuguese_bench, basque_bench, spanish_bench, and galician_bench

- Added duplicate detection in TaskManager._get_task_and_group() that warns users when duplicate task names are found, showing both file paths for easier debugging

- Made directory walk deterministic by sorting dirs and file_list in os.walk() to ensure consistent task loading order across different filesystems and operating systems

The duplicate MMLU-Redux tasks were particularly problematic as they used different datasets but identical names, causing silent conflicts where users would unknowingly run the wrong benchmark variant.

* Fix tags and task names in group info.

Author: giuliolovisotto

lm_eval/tasks/__init__.py

@@ -490,6 +490,8 @@ def _populate_tags_and_groups(config, task, tasks_and_groups, print_info):
         tasks_and_groups = collections.defaultdict()
         for root, dirs, file_list in os.walk(task_dir):
             dirs[:] = [d for d in dirs if d not in ignore_dirs]
+            dirs.sort()  # Sort directories for deterministic traversal order
+            file_list.sort()  # Sort files for consistent processing order
             for f in file_list:
                 if f.endswith(".yaml"):
                     yaml_path = os.path.join(root, f)
@@ -528,6 +530,13 @@ def _populate_tags_and_groups(config, task, tasks_and_groups, print_info):
                     elif self._config_is_task(config):
                         # This is a task config
                         task = config["task"]
+                        if task in tasks_and_groups:
+                            eval_logger.warning(
+                                f"Duplicate task name '{task}' found. "
+                                f"Already registered from: {tasks_and_groups[task]['yaml_path']}. "
+                                f"Skipping duplicate from: {yaml_path}"
+                            )
+                            continue
                         tasks_and_groups[task] = {
                             "type": "task",
                             "yaml_path": yaml_path,

lm_eval/tasks/basque_bench/flores_eu/create_yamls_flores_eu.py

@@ -294,7 +294,7 @@ def gen_lang_yamls(output_dir: str, overwrite: bool) -> None:
                             #                             "group": [f"{BENCH_NAME}_bench", f"{BENCH_NAME}_bench_flores"],
                             #                            "group": "flores_eu",
                             "include": "_flores_common_yaml",
-                            "task": f"flores_{lang_pair_name}",
+                            "task": f"basque_bench_flores_{lang_pair_name}",
                             "doc_to_text": doc_to_text(src, tgt),
                             "doc_to_target": doc_to_target(tgt),
                         },

lm_eval/tasks/basque_bench/flores_eu/flores_ca-eu.yaml

@@ -1,6 +1,6 @@
 # File generated by `create-yamls.py`
 include: _flores_common_yaml
-task: flores_ca-eu
+task: basque_bench_flores_ca-eu
 doc_to_text: 'Catalan sentence: {{sentence_cat_Latn}}
 
   Basque sentence:'

lm_eval/tasks/basque_bench/flores_eu/flores_de-eu.yaml

@@ -1,6 +1,6 @@
 # File generated by `create-yamls.py`
 include: _flores_common_yaml
-task: flores_de-eu
+task: basque_bench_flores_de-eu
 doc_to_text: 'German sentence: {{sentence_deu_Latn}}
 
   Basque sentence:'

lm_eval/tasks/basque_bench/flores_eu/flores_en-eu.yaml

@@ -1,6 +1,6 @@
 # File generated by `create-yamls.py`
 include: _flores_common_yaml
-task: flores_en-eu
+task: basque_bench_flores_en-eu
 doc_to_text: 'English sentence: {{sentence_eng_Latn}}
 
   Basque sentence:'

lm_eval/tasks/basque_bench/flores_eu/flores_es-eu.yaml

@@ -1,6 +1,6 @@
 # File generated by `create-yamls.py`
 include: _flores_common_yaml
-task: flores_es-eu
+task: basque_bench_flores_es-eu
 doc_to_text: 'Spanish sentence: {{sentence_spa_Latn}}
 
   Basque sentence:'

lm_eval/tasks/basque_bench/flores_eu/flores_eu-ca.yaml

@@ -1,6 +1,6 @@
 # File generated by `create-yamls.py`
 include: _flores_common_yaml
-task: flores_eu-ca
+task: basque_bench_flores_eu-ca
 doc_to_text: 'Basque sentence: {{sentence_eus_Latn}}
 
   Catalan sentence:'

lm_eval/tasks/basque_bench/flores_eu/flores_eu-de.yaml

@@ -1,6 +1,6 @@
 # File generated by `create-yamls.py`
 include: _flores_common_yaml
-task: flores_eu-de
+task: basque_bench_flores_eu-de
 doc_to_text: 'Basque sentence: {{sentence_eus_Latn}}
 
   German sentence:'

lm_eval/tasks/basque_bench/flores_eu/flores_eu-en.yaml

@@ -1,6 +1,6 @@
 # File generated by `create-yamls.py`
 include: _flores_common_yaml
-task: flores_eu-en
+task: basque_bench_flores_eu-en
 doc_to_text: 'Basque sentence: {{sentence_eus_Latn}}
 
   English sentence:'

lm_eval/tasks/basque_bench/flores_eu/flores_eu-es.yaml

@@ -1,6 +1,6 @@
 # File generated by `create-yamls.py`
 include: _flores_common_yaml
-task: flores_eu-es
+task: basque_bench_flores_eu-es
 doc_to_text: 'Basque sentence: {{sentence_eus_Latn}}
 
   Spanish sentence:'