Skip to content

Commit

Permalink
fix(squad): use StringSchema instead of MultipleChoiceSchemaLower
Browse files Browse the repository at this point in the history
Switch to StringSchema in the SQuAD benchmark to correctly process
extractive answers.
  • Loading branch information
carvalho28 committed Mar 9, 2025
1 parent 2ae493d commit 8e85b8d
Showing 1 changed file with 3 additions and 3 deletions.
6 changes: 3 additions & 3 deletions deepeval/benchmarks/squad/squad.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from deepeval.models import DeepEvalBaseLLM
from deepeval.benchmarks.squad.task import SQuADTask
from deepeval.benchmarks.squad.template import SQuADTemplate
from deepeval.benchmarks.schema import MultipleChoiceSchemaLower
from deepeval.benchmarks.schema import StringSchema
from deepeval.telemetry import capture_benchmark_run
from deepeval.metrics.utils import initialize_model

Expand Down Expand Up @@ -132,8 +132,8 @@ def predict(self, model: DeepEvalBaseLLM, golden: Golden) -> Dict:

# Enforced model generation
try:
res: MultipleChoiceSchemaLower = model.generate(
prompt=prompt, schema=MultipleChoiceSchemaLower
res: StringSchema = model.generate(
prompt=prompt, schema=StringSchema
)
prediction = res.answer
except TypeError:
Expand Down

0 comments on commit 8e85b8d

Please sign in to comment.