From 8a9eaa632c82d67102f4bc38dbd1095f57b3b99a Mon Sep 17 00:00:00 2001 From: "z.zhong" Date: Thu, 14 Nov 2024 00:12:46 +0900 Subject: [PATCH 1/3] Async call of OpenAISimilarityScorer is slower than sync version Fixes #160 --- src/langcheck/metrics/eval_clients/_openai.py | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/langcheck/metrics/eval_clients/_openai.py b/src/langcheck/metrics/eval_clients/_openai.py index 50c75c03..8b5b2210 100644 --- a/src/langcheck/metrics/eval_clients/_openai.py +++ b/src/langcheck/metrics/eval_clients/_openai.py @@ -409,26 +409,25 @@ def __init__( self.openai_args = openai_args self._use_async = use_async + async def _async_embed(self, inputs: list[str]) -> torch.Tensor: + if self.openai_args: + responses = await self.openai_client.embeddings.create( + input=inputs, **self.openai_args + ) + else: + responses = await self.openai_client.embeddings.create( + input=inputs, model="text-embedding-3-small" + ) + return responses + def _embed(self, inputs: list[str]) -> torch.Tensor: """Embed the inputs using the OpenAI API.""" # TODO: Fix that this async call could be much slower than the sync # version. https://github.com/citadel-ai/langcheck/issues/160 if self._use_async: - - async def _call_async_api() -> Any: - assert isinstance(self.openai_client, AsyncOpenAI) - if self.openai_args: - responses = await self.openai_client.embeddings.create( - input=inputs, **self.openai_args - ) - else: - responses = await self.openai_client.embeddings.create( - input=inputs, model="text-embedding-3-small" - ) - return responses - - embed_response = asyncio.run(_call_async_api()) + assert isinstance(self.openai_client, AsyncOpenAI) + embed_response = asyncio.get_event_loop().run_until_complete(self._async_embed(inputs)) embeddings = [item.embedding for item in embed_response.data] else: From f52eac86dea260eabc122f22eadb28f088d7e153 Mon Sep 17 00:00:00 2001 From: "z.zhong" Date: Fri, 15 Nov 2024 21:18:15 +0900 Subject: [PATCH 2/3] Apply review suggestions. Fixes #160 --- src/langcheck/metrics/eval_clients/_openai.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/langcheck/metrics/eval_clients/_openai.py b/src/langcheck/metrics/eval_clients/_openai.py index 8b5b2210..7cddd424 100644 --- a/src/langcheck/metrics/eval_clients/_openai.py +++ b/src/langcheck/metrics/eval_clients/_openai.py @@ -14,7 +14,7 @@ from ..prompts._utils import get_template from ..scorer._base import BaseSimilarityScorer from ._base import EvalClient, TextResponseWithLogProbs - +from openai.types.create_embedding_response import CreateEmbeddingResponse class OpenAIEvalClient(EvalClient): """EvalClient defined for OpenAI API.""" @@ -409,7 +409,9 @@ def __init__( self.openai_args = openai_args self._use_async = use_async - async def _async_embed(self, inputs: list[str]) -> torch.Tensor: + async def _async_embed(self, inputs: list[str]) -> CreateEmbeddingResponse: + """Embed the inputs using the OpenAI API in async mode.""" + assert isinstance(self.openai_client, AsyncOpenAI) if self.openai_args: responses = await self.openai_client.embeddings.create( input=inputs, **self.openai_args @@ -426,10 +428,13 @@ def _embed(self, inputs: list[str]) -> torch.Tensor: # TODO: Fix that this async call could be much slower than the sync # version. https://github.com/citadel-ai/langcheck/issues/160 if self._use_async: - assert isinstance(self.openai_client, AsyncOpenAI) - embed_response = asyncio.get_event_loop().run_until_complete(self._async_embed(inputs)) - embeddings = [item.embedding for item in embed_response.data] - + try: + loop = asyncio.get_event_loop() + except RuntimeError: # pragma: py-lt-310 + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + embed_response = loop.run_until_complete(self._async_embed(inputs)) + embeddings = [item.embedding for item in embed_response.data] else: assert isinstance(self.openai_client, OpenAI) From 6168eabfb1b714747e65c9a10261c5797d6afbd1 Mon Sep 17 00:00:00 2001 From: Koki Ryu Date: Wed, 20 Nov 2024 03:31:27 +0000 Subject: [PATCH 3/3] Organize the import --- src/langcheck/metrics/eval_clients/_openai.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/langcheck/metrics/eval_clients/_openai.py b/src/langcheck/metrics/eval_clients/_openai.py index 7cddd424..4fa18c84 100644 --- a/src/langcheck/metrics/eval_clients/_openai.py +++ b/src/langcheck/metrics/eval_clients/_openai.py @@ -8,13 +8,14 @@ import torch from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI +from openai.types.create_embedding_response import CreateEmbeddingResponse from langcheck.utils.progress_bar import tqdm_wrapper from ..prompts._utils import get_template from ..scorer._base import BaseSimilarityScorer from ._base import EvalClient, TextResponseWithLogProbs -from openai.types.create_embedding_response import CreateEmbeddingResponse + class OpenAIEvalClient(EvalClient): """EvalClient defined for OpenAI API."""