fix: Add climate fever v2 (embeddings-benchmark#1873)

* Updated ClimateFEVER dataset with new version

* Adds Fill in the empty metadata.

* Updates the date tuple

* Update class name

Co-authored-by: Kenneth Enevoldsen <[email protected]>

* Update domains

Co-authored-by: Kenneth Enevoldsen <[email protected]>

* Update task_subtypes

* Update annotations_creators for the first version

* Update date

Co-authored-by: Kenneth Enevoldsen <[email protected]>

* Update task subtypes

* Update path

* Update description

---------

Co-authored-by: Kenneth Enevoldsen <[email protected]>
Co-authored-by: Mina Parham <[email protected]>

Author: 3 people

mteb/models/lens_models.py

@@ -3,6 +3,7 @@
 from mteb.model_meta import ModelMeta
 from mteb.models.bge_models import bge_full_data
 
+
 lens_d4000 = ModelMeta(
     loader=None,  # TODO: implement this in the future
     name="yibinlei/LENS-d4000",

mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py

@@ -20,7 +20,7 @@ class ClimateFEVER(AbsTaskRetrieval):
         eval_splits=["test"],
         eval_langs=["eng-Latn"],
         main_score="ndcg_at_10",
-        date=None,
+        date=("2001-01-01", "2020-12-31"),  # launch of wiki -> paper publication
         domains=["Encyclopaedic", "Written"],
         task_subtypes=["Claim verification"],
         license="cc-by-sa-4.0",
@@ -56,7 +56,7 @@ class ClimateFEVERHardNegatives(AbsTaskRetrieval):
         eval_splits=["test"],
         eval_langs=["eng-Latn"],
         main_score="ndcg_at_10",
-        date=None,
+        date=("2001-01-01", "2020-12-31"),  # launch of wiki -> paper publication
         domains=["Encyclopaedic", "Written"],
         task_subtypes=["Claim verification"],
         license="cc-by-sa-4.0",
@@ -72,3 +72,39 @@ class ClimateFEVERHardNegatives(AbsTaskRetrieval):
       primaryClass={cs.CL}
 }""",
     )
+
+
+class ClimateFEVERRetrievalv2(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="ClimateFEVER.v2",
+        description="CLIMATE-FEVER is a dataset following the FEVER methodology, containing 1,535 real-world climate change claims. This updated version addresses corpus mismatches and qrel inconsistencies in MTEB, restoring labels while refining corpus-query alignment for better accuracy. ",
+        reference="https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html",
+        dataset={
+            "path": "mteb/climate-fever-v2",
+            "revision": "e438c9586767800aeb10dbe8a245c41dbea4e5f4",
+        },
+        type="Retrieval",
+        category="s2p",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["eng-Latn"],
+        main_score="ndcg_at_10",
+        date=("2001-01-01", "2020-12-31"),  # launch of wiki -> paper publication
+        domains=["Academic", "Written"],
+        task_subtypes=["Claim verification"],
+        license="cc-by-sa-4.0",
+        annotations_creators="human-annotated",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation="""@misc{diggelmann2021climatefever,
+      title={CLIMATE-FEVER: A Dataset for Verification of Real-World Climate Claims}, 
+      author={Thomas Diggelmann and Jordan Boyd-Graber and Jannis Bulian and Massimiliano Ciaramita and Markus Leippold},
+      year={2021},
+      eprint={2012.00614},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}""",
+        prompt={
+            "query": "Given a claim about climate change, retrieve documents that support or refute the claim"
+        },
+    )