update quality checking in rewriter to make it available for cheaper …

…models (gpt-4o-mini)
confident-ai · Mar 2, 2025 · dca20e6 · dca20e6
1 parent b9d7a52
commit dca20e6
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 8 deletions.
diff --git a/deepeval/metrics/utils.py b/deepeval/metrics/utils.py
@@ -260,7 +260,9 @@ def trimAndLoadJson(
         error_str = "Evaluation LLM outputted an invalid JSON. Please use a better evaluation model."
         if metric is not None:
             metric.error = error_str
-        raise ValueError(error_str)
+        print(error_str)
+        return None
+        # raise ValueError(error_str)
     except Exception as e:
         raise Exception(f"An unexpected error occurred: {str(e)}")
 

diff --git a/deepeval/synthesizer/synthesizer.py b/deepeval/synthesizer/synthesizer.py
@@ -763,13 +763,17 @@ async def _a_rewrite_inputs(
                 rewrite_prompt = SynthesizerTemplate.rewrite_synthetic_inputs(
                     context, input, feedback
                 )
-                rewritten_res: RewrittenInput = await self._a_generate_schema(
-                    rewrite_prompt,
-                    RewrittenInput,
-                    self.model,
+                rewritten_res: Optional[RewrittenInput] = (
+                    await self._a_generate_schema(
+                        rewrite_prompt,
+                        RewrittenInput,
+                        self.model,
+                    )
                 )
-                input = rewritten_res.rewritten_input
-
+                input = rewritten_res.rewritten_input if rewritten_res else None
+            # Filter out invalid json directly
+            if not input:
+                continue
             scores.append(score)
             filtered_inputs.append(SyntheticData(input=input))
 
@@ -930,12 +934,14 @@ async def _a_generate_schema(
         prompt: str,
         schema: BaseModel,
         model: DeepEvalBaseLLM,
-    ) -> BaseModel:
+    ) -> Optional[BaseModel]:
         if isinstance(model, GPTModel):
             res, cost = await model.a_generate(prompt)
             if self.synthesis_cost is not None:
                 self.synthesis_cost += cost
             data = trimAndLoadJson(res, self)
+            if not data:
+                return None
             if schema == SyntheticDataList:
                 data_list = [SyntheticData(**item) for item in data["data"]]
                 return SyntheticDataList(data=data_list)