Fix mmlu bpb bug only scoring answer=A questions

allenai · Sep 6, 2024 · 138cfe9 · 138cfe9
1 parent ca81901
commit 138cfe9
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/olmo/eval/downstream.py b/olmo/eval/downstream.py
@@ -1365,6 +1365,9 @@ def doc_to_continuations(self, doc):
             return choices
 
     def doc_to_label(self, doc):
+        if self.metric_type in ["ce_loss", "bpb"]:
+            # Only the correct answer is provided for these metrics
+            return 0
         return doc["answer"]
 
     def doc_to_domain_conditional(self, doc):