diff --git a/deepeval/metrics/metric.py b/deepeval/metrics/metric.py index 2639c1bf8..d1024737c 100644 --- a/deepeval/metrics/metric.py +++ b/deepeval/metrics/metric.py @@ -13,11 +13,11 @@ ) from abc import abstractmethod from ..client import Client -from ..api import Api from ..utils import softmax +from ..singleton import Singleton -class Metric: +class Metric(metaclass=Singleton): # set an arbitrary minimum score that will get over-ridden later minimum_score: float = 0 diff --git a/docs/assets/llm-evaluation-framework.png b/docs/assets/llm-evaluation-framework.png index 6d066fae2..cd638390f 100644 Binary files a/docs/assets/llm-evaluation-framework.png and b/docs/assets/llm-evaluation-framework.png differ diff --git a/docs/docs/framework.md b/docs/docs/framework.md new file mode 100644 index 000000000..d1f1fc371 --- /dev/null +++ b/docs/docs/framework.md @@ -0,0 +1,39 @@ +# About Our Framework + +We segmented LLMs into 4 components - where we are interested in testing the relationship between each component. + +- Query (what the user asks the LLM to do) +- Output (What the LLM returns) +- Context (The context required for the LLM to return a correct response) +- Ethical Considerations (bias, toxicity, etc.) + +:::note + +Over time, we may be updating this framework as we learn more about this field but we are currently presenting this to help new users navigate the large number of metrics that will get added over time. + +::: + +## Why is testing for LLMs so hard? + +LLMs present a paradigm shift in how applications work. They present unique challenges because: + +- Long-tailed nature of queries can mean the answers and queries can be anything +- The answers are stochastic and can be different each time + +DeepEval provides an opinionated way to test these LLM outputs using ML models combined with traditional metrics. While there is inherent circularity in using deep learning to test deep learning model outputs, we believe that this type of approach can provide better guides than other methods into which areas to firstly investigate. + +Below, we share our framework for large language models. + +![Evaluation Framework](../assets/llm-evaluation-framework.png) + +## Why not use GPT? + +GPT itself is an auto-regressive model that can provide evaluation of the model. GPT aims to predict the next best token and can provide a score based on that. However, imagine the following situation: + +GPT predicts 0.2. +It then needs to predict the next number based on the given output. +GPT then predicts 0.23 +It then needs to predict the next number based on the given output. +GPT then predicts 0.235 + +The task of predicting the next number does not seem to provide a strong enough reason to get this working. diff --git a/docs/docs/quickstart/custom-metrics.md b/docs/docs/quickstart/custom-metrics.md index 05b19979c..3f8670436 100644 --- a/docs/docs/quickstart/custom-metrics.md +++ b/docs/docs/quickstart/custom-metrics.md @@ -1,12 +1,12 @@ # Define Your Own Metric -By default, we support the following metrics: +You can define a custom metric by defining the `measure` and `is_successful` functions and inheriting the base `Metric` class. An example is provided below. -- BertScoreMetric (simply set `metric="BertScoreMetric"`) -- Entailment Score (simply set `metric="entailment"`) -- Exact string match (simply set `metric="exact"`) +:::note -You can define a custom metric by defining the `measure` and `is_successful` functions and inheriting the base `Metric` class. An example is provided below. +As of right now, we do not currently support custom metrics for our dashboard but this will be supported in an upcoming version - we apologise for the wait! For any requests of additional metrics, please feel free to e-mail jacky@confident-ai.com + +::: ```python import asyncio @@ -21,15 +21,7 @@ class LengthMetric(Metric): # sends to server score = self.measure(text) # Optional: Logs it to the server - asyncio.create_task( - self._send_to_server( - metric_score=score, - metric_name=self.__name__, - query=text, - success = self.success - ) - ) - return self.measure(text) + return score def measure(self, text: str): self.success = len(x) > self.minimum_length diff --git a/docs/docs/quickstart/quickstart.md b/docs/docs/quickstart/quickstart.md index 5d25d05f9..ca9598224 100644 --- a/docs/docs/quickstart/quickstart.md +++ b/docs/docs/quickstart/quickstart.md @@ -29,19 +29,36 @@ def generate_chatgpt_output(query: str): def test_factual_consistency(): query = "What is the customer success phone line?" - expected_output = "Our customer success phone line is 1200-231-231." + context = "Our customer success phone line is 1200-231-231." output = generate_chatgpt_output(query) - assert_factual_consistency(output, expected_output) + assert_factual_consistency(output, context) +# Just run the following code in Python if required test_factual_consistency() ``` -You can then run it in CLI using this: +### Running it in Pytest + +To run this in Pytest, you will need the asyncio framework ```bash -python -m pytest test_sample.py +pip install pytest-asyncio ``` -## Writing a custom metric +```python +# sample.py +import pytest -With `deepeval`, you can easily set custom metrics or customize existing metrics. We recommend reading the `Define Your Own Metric` if you are. +@pytest.mark.asyncio +async def test_factual_consistency(): + query = "What is the customer success phone line?" + context = "Our customer success phone line is 1200-231-231." + output = generate_chatgpt_output(query) + assert_factual_consistency(output, context) +``` + +You can then run it in CLI using: + +```bash +python -m pytest test_sample.py +``` diff --git a/docs/sidebars.js b/docs/sidebars.js index ec437401b..a05a7cb72 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -20,6 +20,7 @@ const sidebars = { tutorialSidebar: [ 'index', 'install', + 'framework', { type: 'category', label: 'QuickStart',