Dataset/update climate fever

mteb/models/gme_models.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import logging
-from functools import partial
 
 from mteb.model_meta import ModelMeta
 

mteb/models/lens_models.py

@@ -1,12 +1,6 @@
 from __future__ import annotations
 
-from functools import partial
-
-import torch
-
-from mteb.encoder_interface import PromptType
-from mteb.model_meta import ModelMeta, sentence_transformers_loader
-from mteb.models.instruct_wrapper import instruct_wrapper
+from mteb.model_meta import ModelMeta
 
 from .bge_models import bge_full_data
 

mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py

@@ -60,7 +60,7 @@ class ClimateFEVERHardNegatives(AbsTaskRetrieval):
         domains=None,
         task_subtypes=None,
         license=None,
-        annotations_creators=None,
+        annotations_creators="human-annotated",
         dialect=None,
         sample_creation=None,
         bibtex_citation="""@misc{diggelmann2021climatefever,
@@ -72,3 +72,39 @@ class ClimateFEVERHardNegatives(AbsTaskRetrieval):
       primaryClass={cs.CL}
 }""",
     )
+
+
+class ClimateFEVERRetrievalv2(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="ClimateFEVER.v2",
+        description="CLIMATE-FEVER is a dataset adopting the FEVER methodology that consists of 1,535 real-world claims regarding climate-change. ",
+        reference="https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html",
+        dataset={
+            "path": "mteb/climate-fever-v2",
+            "revision": "e438c9586767800aeb10dbe8a245c41dbea4e5f4",
+        },
+        type="Retrieval",
+        category="s2p",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["eng-Latn"],
+        main_score="ndcg_at_10",
+        date=("2001-01-01", "2020-12-31"), # launch of wiki -> paper publication
+        domains=["Academic", "Written"],
+        task_subtypes=["Claim verification"],
+        license="cc-by-sa-4.0",
+        annotations_creators="human-annotated",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation="""@misc{diggelmann2021climatefever,
+      title={CLIMATE-FEVER: A Dataset for Verification of Real-World Climate Claims}, 
+      author={Thomas Diggelmann and Jordan Boyd-Graber and Jannis Bulian and Massimiliano Ciaramita and Markus Leippold},
+      year={2021},
+      eprint={2012.00614},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}""",
+        prompt={
+            "query": "Given a claim about climate change, retrieve documents that support or refute the claim"
+        },
+    )