Refactor SummaryRetrieval to subclass BitextMining

embeddings-benchmark · Jan 29, 2025 · a6ac99b · a6ac99b
1 parent c5a611b
commit a6ac99b
Show file tree

Hide file tree

Showing 8 changed files with 19 additions and 369 deletions.
diff --git a/mteb/abstasks/AbsTaskSummaryRetrieval.py b/mteb/abstasks/AbsTaskSummaryRetrieval.py
diff --git a/mteb/abstasks/__init__.py b/mteb/abstasks/__init__.py
@@ -13,5 +13,4 @@
 from .AbsTaskSpeedTask import *
 from .AbsTaskSTS import *
 from .AbsTaskSummarization import *
-from .AbsTaskSummaryRetrieval import *
 from .MultilingualTask import *
diff --git a/mteb/benchmarks/benchmarks.py b/mteb/benchmarks/benchmarks.py
@@ -1329,6 +1329,7 @@ def load_results(
     description="Main Persian (Farsi) benchmarks from MTEB",
     reference=None,
     citation=None,
+    contacts=['mehran-sarmadi', 'ERfun', 'morteza20'],
 )
 
 CHEMTEB = Benchmark(

diff --git a/mteb/evaluation/evaluators/SummaryRetrievalEvaluator.py b/mteb/evaluation/evaluators/SummaryRetrievalEvaluator.py
diff --git a/mteb/evaluation/evaluators/__init__.py b/mteb/evaluation/evaluators/__init__.py
@@ -7,5 +7,4 @@
 from .RerankingEvaluator import *
 from .RetrievalEvaluator import *
 from .STSEvaluator import *
-from .SummarizationEvaluator import *
-from .SummaryRetrievalEvaluator import *
+from .SummarizationEvaluator import *
diff --git a/mteb/tasks/SummaryRetrieval/fas/FaMTEBSummaryRetrieval.py b/mteb/tasks/SummaryRetrieval/fas/FaMTEBSummaryRetrieval.py
@@ -1,10 +1,10 @@
 from __future__ import annotations
 
-from mteb.abstasks.AbsTaskSummaryRetrieval import AbsTaskSummaryRetrieval
+from mteb.abstasks.AbsTaskBitextMining import AbsTaskBitextMining
 from mteb.abstasks.TaskMetadata import TaskMetadata
 
 
-class SAMSumFa(AbsTaskSummaryRetrieval):
+class SAMSumFa(AbsTaskBitextMining):
     metadata = TaskMetadata(
         name="SAMSumFa",
         description="Translated Version of SAMSum Dataset",
@@ -28,9 +28,13 @@ class SAMSumFa(AbsTaskSummaryRetrieval):
         sample_creation="found",
         bibtex_citation=""" """,
     )
+    def dataset_transform(self):
+        self.dataset = self.dataset.rename_columns(
+            {"text": "sentence1", "summary": "sentence2"}
+        )
 
 
-class SynPerChatbotSumSRetrieval(AbsTaskSummaryRetrieval):
+class SynPerChatbotSumSRetrieval(AbsTaskBitextMining):
     metadata = TaskMetadata(
         name="SynPerChatbotSumSRetrieval",
         description="Synthetic Persian Chatbot  Summary Dataset",
@@ -54,9 +58,12 @@ class SynPerChatbotSumSRetrieval(AbsTaskSummaryRetrieval):
         sample_creation="LM-generated and verified",
         bibtex_citation=""" """,
     )
+    def dataset_transform(self):
+        self.dataset = self.dataset.rename_columns(
+            {"text": "sentence1", "summary": "sentence2"}
+        )
 
-
-class SynPerChatbotRAGSumSRetrieval(AbsTaskSummaryRetrieval):
+class SynPerChatbotRAGSumSRetrieval(AbsTaskBitextMining):
     metadata = TaskMetadata(
         name="SynPerChatbotRAGSumSRetrieval",
         description="Synthetic Persian Chatbot RAG Summary Dataset",
@@ -80,3 +87,7 @@ class SynPerChatbotRAGSumSRetrieval(AbsTaskSummaryRetrieval):
         sample_creation="LM-generated and verified",
         bibtex_citation=""" """,
     )
+    def dataset_transform(self):
+        self.dataset = self.dataset.rename_columns(
+            {"text": "sentence1", "summary": "sentence2"}
+        )