Custom LLM in the docs provide no schema input for generate method, and returns string, which causes summarization.py raise AttributeError: 'str' object has no attribute 'truths'

[pin-lpt-heatherwick]

`from deepeval.models.base_model import DeepEvalBaseLLM

class Mistral7B(DeepEvalBaseLLM):
def init(
self,
model,
tokenizer
):
self.model = model
self.tokenizer = tokenizer

def load_model(self):
    return self.model

def generate(self, prompt: str, schema=None) -> str:

    if not prompt or len(prompt.strip()) == 0:
        raise ValueError("Prompt cannot be empty")

    model = self.load_model()
    device = "cuda" 

    model_inputs = self.tokenizer([prompt], return_tensors="pt").to(device)
    model.to(device)

    if model_inputs.input_ids.numel() == 0:
        raise ValueError("Tokenized input is empty")

    generated_ids = model.generate(
        **model_inputs, 
        max_new_tokens=2000, 
        do_sample=True,
        pad_token_id=self.tokenizer.eos_token_id
        eos_token_id=self.tokenizer.eos_token_id
    )
    response = self.tokenizer.batch_decode(generated_ids)[0]
    print('response is...\n', response)
    
    return response

async def a_generate(self, prompt: str, schema=None) -> str:
    return self.generate(prompt, schema)


def get_model_name(self):
    return "Mistral 7B"`

Traceback (most recent call last):
File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/gradio/queueing.py", line 625, in process_events
response = await route_utils.call_process_api(
File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/gradio/route_utils.py", line 322, in call_process_api
output = await app.get_blocks().process_api(
File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/gradio/blocks.py", line 2108, in process_api
result = await self.call_function(
File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/gradio/blocks.py", line 1655, in call_function
prediction = await anyio.to_thread.run_sync( # type: ignore
File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/anyio/to_thread.py", line 56, in run_sync
return await get_async_backend().run_sync_in_worker_thread(
File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 2461, in run_sync_in_worker_thread
return await future
File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 962, in run
result = context.run(func, *args)
File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/gradio/utils.py", line 890, in wrapper
response = f(*args, **kwargs)
File "/home/user/app/app.py", line 66, in run_metric
metric.measure(test_case)
File "/home/user/app/deepeval/metrics/summarization/summarization.py", line 76, in measure
loop.run_until_complete(
File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete
return future.result()
File "/home/user/app/deepeval/metrics/summarization/summarization.py", line 128, in a_measure
self.truths, self.claims = await asyncio.gather(
File "/home/user/app/deepeval/metrics/summarization/summarization.py", line 489, in _a_generate_truths
return res.truths
AttributeError: 'str' object has no attribute 'truths'

[penguine-ip]

@pin-lpt-heatherwick because you didn't return a pydantic BaseModel as is required when you define a schema argument. Either remove the schema argument or return a BaseModel.

[pin-lpt-heatherwick]

I tried using the doc as is already as below

    def generate(self, prompt: str) -> str:
        model = self.load_model()

        device = "cuda" # the device to load the model onto

        model_inputs = self.tokenizer([prompt], return_tensors="pt").to(device)
        model.to(device)

        generated_ids = model.generate(**model_inputs, max_new_tokens=100, do_sample=True)
        return self.tokenizer.batch_decode(generated_ids)[0]

    async def a_generate(self, prompt: str) -> str:
        return self.generate(prompt)

which raises the error below

Traceback (most recent call last):
File "/home/user/app/deepeval/metrics/summarization/summarization.py", line 488, in _a_generate_truths
res: Truths = await self.model.a_generate(prompt, schema=Truths)
TypeError: Mistral7B.a_generate() got an unexpected keyword argument 'schema'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/home/user/app/deepeval/metrics/utils.py", line 258, in trimAndLoadJson
return json.loads(jsonStr)
File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/json/init.py", line 346, in loads
return _default_decoder.decode(s)
File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/json/decoder.py", line 353, in raw_decode
obj, end = self.scan_once(s, idx)
json.decoder.JSONDecodeError: Expecting ',' delimiter: line 4 column 9 (char 120)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/gradio/queueing.py", line 625, in process_events
response = await route_utils.call_process_api(
File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/gradio/route_utils.py", line 322, in call_process_api
output = await app.get_blocks().process_api(
File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/gradio/blocks.py", line 2108, in process_api
result = await self.call_function(
File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/gradio/blocks.py", line 1655, in call_function
prediction = await anyio.to_thread.run_sync( # type: ignore
File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/anyio/to_thread.py", line 56, in run_sync
return await get_async_backend().run_sync_in_worker_thread(
File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 2461, in run_sync_in_worker_thread
return await future
File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 962, in run
result = context.run(func, *args)
File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/gradio/utils.py", line 890, in wrapper
response = f(*args, **kwargs)
File "/home/user/app/app.py", line 66, in run_metric
metric.measure(test_case)
File "/home/user/app/deepeval/metrics/summarization/summarization.py", line 76, in measure
loop.run_until_complete(
File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete
return future.result()
File "/home/user/app/deepeval/metrics/summarization/summarization.py", line 128, in a_measure
self.truths, self.claims = await asyncio.gather(
File "/home/user/app/deepeval/metrics/summarization/summarization.py", line 492, in _a_generate_truths
data = trimAndLoadJson(res, self)
File "/home/user/app/deepeval/metrics/utils.py", line 263, in trimAndLoadJson
raise ValueError(error_str)
ValueError: Evaluation LLM outputted an invalid JSON. Please use a better evaluation model.

[penguine-ip]

@pin-lpt-heatherwick hmm i see, let me check back to you tomorrow on this metric, thanks for flagging it