From 036b894456ed53256d383b6d92d9bea957cf8bb7 Mon Sep 17 00:00:00 2001 From: James Braza Date: Tue, 14 Jan 2025 11:30:54 -0800 Subject: [PATCH 1/2] Incorporated CommonLLMNames from llmclient --- ldp/agent/__init__.py | 10 - ldp/agent/react_agent.py | 5 +- ldp/agent/simple_agent.py | 5 +- ldp/agent/tree_of_thoughts_agent.py | 5 +- tests/__init__.py | 9 - ...stLLMCallOp.test_compute_logprob[0.0].yaml | 40 +- ...stLLMCallOp.test_compute_logprob[0.5].yaml | 34 +- ...stLLMCallOp.test_compute_logprob[1.0].yaml | 34 +- ...TestReActAgent.test_multi_step[False].yaml | 457 ++++++++++-------- .../TestReActAgent.test_multi_step[True].yaml | 169 +++---- tests/test_agents.py | 21 +- tests/test_envs.py | 5 +- tests/test_modules.py | 10 +- tests/test_ops.py | 4 +- tests/test_optimizer.py | 4 +- 15 files changed, 434 insertions(+), 378 deletions(-) diff --git a/ldp/agent/__init__.py b/ldp/agent/__init__.py index f248337f..3597d56f 100644 --- a/ldp/agent/__init__.py +++ b/ldp/agent/__init__.py @@ -1,12 +1,3 @@ -from enum import StrEnum - - -class DefaultLLMModelNames(StrEnum): - """Defaults for LLM models, pin exact versions for performance stability.""" - - OPENAI = "gpt-4o-2024-08-06" # Cheap, fast, and decent - - # Lower than LiteLLM's 10-min default: https://github.com/BerriAI/litellm/blob/v1.48.10/litellm/main.py#L859 DEFAULT_LLM_COMPLETION_TIMEOUT = 120 # seconds @@ -23,7 +14,6 @@ class DefaultLLMModelNames(StrEnum): "DEFAULT_LLM_COMPLETION_TIMEOUT", "Agent", "AgentConfig", - "DefaultLLMModelNames", "HTTPAgentClient", "MemoryAgent", "ReActAgent", diff --git a/ldp/agent/react_agent.py b/ldp/agent/react_agent.py index 17eb7bca..027799c6 100644 --- a/ldp/agent/react_agent.py +++ b/ldp/agent/react_agent.py @@ -8,6 +8,7 @@ ToolRequestMessage, ToolResponseMessage, ) +from llmclient import CommonLLMNames from pydantic import BaseModel, ConfigDict, Field from tenacity import ( Future, @@ -29,7 +30,7 @@ ToolDescriptionMethods, ) -from . import DEFAULT_LLM_COMPLETION_TIMEOUT, DefaultLLMModelNames +from . import DEFAULT_LLM_COMPLETION_TIMEOUT from .agent import Agent from .simple_agent import SimpleAgentState @@ -81,7 +82,7 @@ class ReActAgent(BaseModel, Agent[SimpleAgentState]): llm_model: dict[str, Any] = Field( default={ - "model": DefaultLLMModelNames.OPENAI.value, + "model": CommonLLMNames.OPENAI_BASELINE.value, "temperature": 0.1, "logprobs": True, "top_logprobs": 1, diff --git a/ldp/agent/simple_agent.py b/ldp/agent/simple_agent.py index 3d2ef062..ce7b329a 100644 --- a/ldp/agent/simple_agent.py +++ b/ldp/agent/simple_agent.py @@ -4,12 +4,13 @@ from aviary.core import Message, Tool, ToolRequestMessage, ToolResponseMessage from aviary.message import EnvStateMessage +from llmclient import CommonLLMNames from pydantic import BaseModel, ConfigDict, Field from ldp.graph import ConfigOp, LLMCallOp, OpResult, compute_graph from ldp.llms import prepend_sys -from . import DEFAULT_LLM_COMPLETION_TIMEOUT, DefaultLLMModelNames +from . import DEFAULT_LLM_COMPLETION_TIMEOUT from .agent import Agent @@ -83,7 +84,7 @@ class SimpleAgent(BaseModel, Agent[SimpleAgentState]): llm_model: dict[str, Any] = Field( default={ - "model": DefaultLLMModelNames.OPENAI.value, + "model": CommonLLMNames.OPENAI_BASELINE.value, "temperature": 0.1, "timeout": DEFAULT_LLM_COMPLETION_TIMEOUT, }, diff --git a/ldp/agent/tree_of_thoughts_agent.py b/ldp/agent/tree_of_thoughts_agent.py index def3718f..1e232963 100644 --- a/ldp/agent/tree_of_thoughts_agent.py +++ b/ldp/agent/tree_of_thoughts_agent.py @@ -17,12 +17,13 @@ from typing import Any from aviary.core import Message, Tool, ToolCall, ToolRequestMessage +from llmclient import CommonLLMNames from pydantic import BaseModel, ConfigDict, Field from ldp.graph import FxnOp, LLMCallOp, OpResult, compute_graph, get_call_id, op_call from ldp.llms import prepend_sys -from . import DEFAULT_LLM_COMPLETION_TIMEOUT, DefaultLLMModelNames +from . import DEFAULT_LLM_COMPLETION_TIMEOUT from .agent import Agent from .simple_agent import SimpleAgentState @@ -43,7 +44,7 @@ class TreeofThoughtsAgent(BaseModel, Agent[SimpleAgentState]): llm_model: dict[str, Any] = Field( default={ - "model": DefaultLLMModelNames.OPENAI.value, + "model": CommonLLMNames.OPENAI_BASELINE.value, "temperature": 0.1, "timeout": DEFAULT_LLM_COMPLETION_TIMEOUT, }, diff --git a/tests/__init__.py b/tests/__init__.py index 293657e6..912238c3 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,13 +1,4 @@ import pathlib -from enum import StrEnum - - -class CILLMModelNames(StrEnum): - """Models to use for generic CI testing.""" - - ANTHROPIC = "claude-3-haiku-20240307" # Cheap and not Anthropic's cutting edge - OPENAI = "gpt-4o-mini-2024-07-18" # Cheap and not OpenAI's cutting edge - TESTS_DIR = pathlib.Path(__file__).parent CASSETTES_DIR = TESTS_DIR / "cassettes" diff --git a/tests/cassettes/TestLLMCallOp.test_compute_logprob[0.0].yaml b/tests/cassettes/TestLLMCallOp.test_compute_logprob[0.0].yaml index 998aa4bf..a9e0a57b 100644 --- a/tests/cassettes/TestLLMCallOp.test_compute_logprob[0.0].yaml +++ b/tests/cassettes/TestLLMCallOp.test_compute_logprob[0.0].yaml @@ -1,8 +1,8 @@ interactions: - request: body: - '{"messages": [{"role": "user", "content": "Hello"}], "model": "gpt-4o-mini", - "temperature": 0.0}' + '{"messages": [{"role": "user", "content": "Hello"}], "model": "gpt-4o-mini-2024-07-18", + "n": 1, "temperature": 0.0}' headers: accept: - application/json @@ -11,13 +11,13 @@ interactions: connection: - keep-alive content-length: - - "96" + - "115" content-type: - application/json host: - api.openai.com user-agent: - - AsyncOpenAI/Python 1.54.3 + - AsyncOpenAI/Python 1.59.3 x-stainless-arch: - arm64 x-stainless-async: @@ -27,7 +27,7 @@ interactions: x-stainless-os: - MacOS x-stainless-package-version: - - 1.54.3 + - 1.59.3 x-stainless-raw-response: - "true" x-stainless-retry-count: @@ -41,19 +41,19 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAAA4xSQW7bMBC86xVbnq1CclzY8SVoUqApih6SALkUhUCRK5kJxWXJVVsn8N8DSo7l - IAnQCw8zO8OZxT5mAMJosQahNpJV523++fpG33+6+tL5W+Sbi/rH1cU2nNOf718f6t9ilhRU36Hi - Z9VHRZ23yIbcSKuAkjG5lsuTsixWJ+V8IDrSaJOs9ZwvKO+MM/m8mC/yYpmXq716Q0ZhFGv4mQEA - PA5vyuk0/hNrKGbPSIcxyhbF+jAEIALZhAgZo4ksHYvZRCpyjG6IfonW0ge4pL+gpINvMApgSz0w - abk9OxYGbPooU3jXW7vHd4ckllofqI57/oA3xpm4qQLKSC79Gpm8GNhdBvBraNy/KCF8oM5zxXSP - LhmuRjcxrXniTvccE0s7weVy9oZXpZGlsfFoX0JJtUE9Kaflyl4bOiKyo8avs7zlPbY2rv0f+4lQ - Cj2jrnxAbdTLvtNYwHSD740dNjwEFnEbGbuqMa7F4IMZL6DxVVHLQpfzRVOKbJc9AQAA//8DAOqA - rbEPAwAA + H4sIAAAAAAAAA4xSQW7bMBC86xVbnq1Clt3Y9aUIerGvLRAgCAKBJlcyG4rLkqs2RuC/B5RsS0Zb + oBcdZnZGM8t9ywCE0WIDQh0kq9bb/N43r7ufW6Tu8Vv8XET6Tg+tK7T+uii0mCUF7X+g4ovqo6LW + W2RDbqBVQMmYXOerxd16vSjnZU+0pNEmWeM5X1LeGmfysiiXebHK5+uz+kBGYRQbeMoAAN76b8rp + NL6KDRSzC9JijLJBsbkOAYhANiFCxmgiS8diNpKKHKPro2/RWvoAW/oNSjrYwSCAI3XApOXxy1QY + sO6iTOFdZ+0ZP12TWGp8oH0881e8Ns7EQxVQRnLpr5HJi549ZQDPfePupoTwgVrPFdMLumS4HtzE + uOaRm5+XIZhY2gl+Ed2YVRpZGhsnCxNKqgPqUTluV3ba0ITIJpX/DPM376G2cc3/2I+EUugZdeUD + aqNuC49jAdMR/mvsuuI+sIgYfhmFFRsM6Rk01rKzw2mIeIyMbVUb12DwwQz3UftqVaJeyf2npRLZ + KXsHAAD//wMAEDvPji0DAAA= headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8df9522b1bb9176b-SJC + - 90200eeee9d7174e-SJC Connection: - keep-alive Content-Encoding: @@ -61,9 +61,15 @@ interactions: Content-Type: - application/json Date: - - Fri, 08 Nov 2024 23:25:13 GMT + - Tue, 14 Jan 2025 19:33:33 GMT Server: - cloudflare + Set-Cookie: + - __cf_bm=UQ5laTP9Pnzbx_cAQ0fhk0VBG915_xMVIUeFrx2yVx0-1736883213-1.0.1.1-JCTt9uio_P3O5iDe8YaMBuZUFyrhlGDG8S.DahTCxxbe1WsS6hHltTsfVZ0znl.y1nkpb9NSlv5GHuo1SfYd9Q; + path=/; expires=Tue, 14-Jan-25 20:03:33 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=zK8680ITY0A0uyqUIYD4RjgzO5O1h13Zw2DSCcLdaTQ-1736883213044-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None Transfer-Encoding: - chunked X-Content-Type-Options: @@ -75,7 +81,7 @@ interactions: openai-organization: - future-house-xr4tdh openai-processing-ms: - - "308" + - "275" openai-version: - "2020-10-01" strict-transport-security: @@ -93,7 +99,7 @@ interactions: x-ratelimit-reset-tokens: - 0s x-request-id: - - req_578a56bf5fc1a231deb3d6bcba377206 + - req_b67f9d10a1a62ae04ccdc3c7fa8bf5ad status: code: 200 message: OK diff --git a/tests/cassettes/TestLLMCallOp.test_compute_logprob[0.5].yaml b/tests/cassettes/TestLLMCallOp.test_compute_logprob[0.5].yaml index 06a96776..7786da69 100644 --- a/tests/cassettes/TestLLMCallOp.test_compute_logprob[0.5].yaml +++ b/tests/cassettes/TestLLMCallOp.test_compute_logprob[0.5].yaml @@ -1,8 +1,8 @@ interactions: - request: body: - '{"messages": [{"role": "user", "content": "Hello"}], "model": "gpt-4o-mini", - "temperature": 0.5}' + '{"messages": [{"role": "user", "content": "Hello"}], "model": "gpt-4o-mini-2024-07-18", + "n": 1, "temperature": 0.5}' headers: accept: - application/json @@ -11,13 +11,13 @@ interactions: connection: - keep-alive content-length: - - "96" + - "115" content-type: - application/json host: - api.openai.com user-agent: - - AsyncOpenAI/Python 1.54.3 + - AsyncOpenAI/Python 1.59.3 x-stainless-arch: - arm64 x-stainless-async: @@ -27,7 +27,7 @@ interactions: x-stainless-os: - MacOS x-stainless-package-version: - - 1.54.3 + - 1.59.3 x-stainless-raw-response: - "true" x-stainless-retry-count: @@ -41,19 +41,19 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAAA4xSQW7bMBC86xVbnq1CUpza8aXILb3GMAykKASaXMl0KS5BrtI6gf9eUHIsB02B - XniY2RnOLPY1AxBGixUItZesOm/z+8e1ru7W2+cX3j7q9a0+3h82T4et2WyepJglBe0OqPhN9VlR - 5y2yITfSKqBkTK7l4qYsi2X1ZTkQHWm0SdZ6zueUd8aZvCqqeV4s8nJ5Vu/JKIxiBd8zAIDX4U05 - ncbfYgXF7A3pMEbZolhdhgBEIJsQIWM0kaVjMZtIRY7RDdEf0Fr6BA/0C5R08A1GARypByYtj1+v - hQGbPsoU3vXWnvHTJYml1gfaxTN/wRvjTNzXAWUkl36NTF4M7CkD+DE07t+VED5Q57lm+okuGS5H - NzGteeLuzhwTSzvB5WL2gVetkaWx8WpfQkm1Rz0pp+XKXhu6IrKrxn9n+ch7bG1c+z/2E6EUekZd - +4DaqPd9p7GA6Qb/NXbZ8BBYxGNk7OrGuBaDD2a8gMbXxU4WuqzmTSmyU/YHAAD//wMAJKyyfA8D - AAA= + H4sIAAAAAAAAA4xSQW7bMBC86xXbPVuFZQWJ7EuRS5ukQJB7UAg0uZKZUFyCpNK4gf9eULItGW2B + XnSY2RnNLPcjA0CtcAModyLKzpn81rXvD5X5/uur2bN/UrJ4fFwXas3Ft9cHXCQFb19IxpPqs+TO + GYqa7UhLTyJSci1uyuuqKldFORAdKzJJ1rqYX3Heaavz1XJ1lS9v8qI6qnesJQXcwHMGAPAxfFNO + q+gdN7BcnJCOQhAt4eY8BICeTUJQhKBDFDbiYiIl20h2iH5HxvAnuOOfIIWFexgFsOceIiux/zIX + emr6IFJ42xtzxA/nJIZb53kbjvwZb7TVYVd7EoFt+muI7HBgDxnAj6Fxf1ECnefOxTryK9lkWI1u + OK154orjMjByFGaGn0QXZrWiKLQJs4WhFHJHalJO2xW90jwjslnlP8P8zXusrW37P/YTISW5SKp2 + npSWl4WnMU/pCP81dl7xEBgD+TctqY6afHoGRY3ozXgaGPYhUlc32rbkndfjfTSu3qqqLFfr5rrE + 7JD9BgAA//8DAHzPNyctAwAA headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8df95113983fcef5-SJC + - 90200ef4e82ceb20-SJC Connection: - keep-alive Content-Encoding: @@ -61,7 +61,7 @@ interactions: Content-Type: - application/json Date: - - Fri, 08 Nov 2024 23:24:28 GMT + - Tue, 14 Jan 2025 19:33:34 GMT Server: - cloudflare Transfer-Encoding: @@ -75,7 +75,7 @@ interactions: openai-organization: - future-house-xr4tdh openai-processing-ms: - - "283" + - "330" openai-version: - "2020-10-01" strict-transport-security: @@ -93,7 +93,7 @@ interactions: x-ratelimit-reset-tokens: - 0s x-request-id: - - req_9782ec5fccde956ecd7d4c363b44bc91 + - req_309cd40a2787f7b08cfd1d6a7e894490 status: code: 200 message: OK diff --git a/tests/cassettes/TestLLMCallOp.test_compute_logprob[1.0].yaml b/tests/cassettes/TestLLMCallOp.test_compute_logprob[1.0].yaml index 9b161085..93a700ea 100644 --- a/tests/cassettes/TestLLMCallOp.test_compute_logprob[1.0].yaml +++ b/tests/cassettes/TestLLMCallOp.test_compute_logprob[1.0].yaml @@ -1,8 +1,8 @@ interactions: - request: body: - '{"messages": [{"role": "user", "content": "Hello"}], "model": "gpt-4o-mini", - "temperature": 1.0}' + '{"messages": [{"role": "user", "content": "Hello"}], "model": "gpt-4o-mini-2024-07-18", + "n": 1, "temperature": 1.0}' headers: accept: - application/json @@ -11,13 +11,13 @@ interactions: connection: - keep-alive content-length: - - "96" + - "115" content-type: - application/json host: - api.openai.com user-agent: - - AsyncOpenAI/Python 1.54.3 + - AsyncOpenAI/Python 1.59.3 x-stainless-arch: - arm64 x-stainless-async: @@ -27,7 +27,7 @@ interactions: x-stainless-os: - MacOS x-stainless-package-version: - - 1.54.3 + - 1.59.3 x-stainless-raw-response: - "true" x-stainless-retry-count: @@ -41,19 +41,19 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAAA4xSXWvcMBB896/Y6vlc7Luj9/ESjkBJSOhDQgslFKOT1rYaWSukNe0R7r8X2Zfz - habQFz3M7Ixmln3JAITRYgtCtZJV522+e3jUi89fNt923xfX9/Lhsfm6snehXe9M3YlZUtD+Jyp+ - VX1U1HmLbMiNtAooGZNruVqUZbGef9oMREcabZI1nvMl5Z1xJp8X82VerPJyfVK3ZBRGsYWnDADg - ZXhTTqfxt9hCMXtFOoxRNii25yEAEcgmRMgYTWTpWMwmUpFjdEP0G7SWPsAN/QIlHdzCKIAD9cCk - 5eHqUhiw7qNM4V1v7Qk/npNYanygfTzxZ7w2zsS2CigjufRrZPJiYI8ZwI+hcf+mhPCBOs8V0zO6 - ZLge3cS05onbnDgmlnaCy9XsHa9KI0tj48W+hJKqRT0pp+XKXhu6ILKLxn9nec97bG1c8z/2E6EU - ekZd+YDaqLd9p7GA6Qb/NXbe8BBYxENk7KrauAaDD2a8gNpXxV4Wupwv61Jkx+wPAAAA//8DAC+9 - 7CsPAwAA + H4sIAAAAAAAAAwAAAP//jFJBbtswELzrFVuercK2DFvxpeihgIPklBbooSgEmlxJTCkuQa6SGIH/ + XlByLAdJgFx0mNkZzSz3OQMQRostCNVKVp23+XffPN0UptmpH79/3t7fXt9R+SDXq8f9rv0lZklB + +3tU/KL6qqjzFtmQG2kVUDIm18WmWJdlsVysBqIjjTbJGs/5ivLOOJMv58tVPt/ki/KkbskojGIL + fzIAgOfhm3I6jU9iC/PZC9JhjLJBsT0PAYhANiFCxmgiS8diNpGKHKMbou/QWvoCO3oEJR1cwyiA + A/XApOXh26UwYN1HmcK73toTfjwnsdT4QPt44s94bZyJbRVQRnLpr5HJi4E9ZgB/h8b9qxLCB+o8 + V0z/0CXDcnQT05on7urEMbG0E7zYzN7xqjSyNDZe7EsoqVrUk3Jaruy1oQsiu2j8Nst73mNr45rP + 2E+EUugZdeUDaqNe953GAqYb/GjsvOEhsIgYHozCig2G9Aoaa9nb8TJEPETGrqqNazD4YMbzqH21 + 12VRLK/qdSGyY/YfAAD//wMA7U+cOiwDAAA= headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8df951193f79fafc-SJC + - 90200efb0f5feb26-SJC Connection: - keep-alive Content-Encoding: @@ -61,7 +61,7 @@ interactions: Content-Type: - application/json Date: - - Fri, 08 Nov 2024 23:24:29 GMT + - Tue, 14 Jan 2025 19:33:35 GMT Server: - cloudflare Transfer-Encoding: @@ -75,7 +75,7 @@ interactions: openai-organization: - future-house-xr4tdh openai-processing-ms: - - "272" + - "399" openai-version: - "2020-10-01" strict-transport-security: @@ -93,7 +93,7 @@ interactions: x-ratelimit-reset-tokens: - 0s x-request-id: - - req_8cba5ef357c8edab9b2a215420f0dbf5 + - req_97dba0dc17a8d2572fe30083e06468b0 status: code: 200 message: OK diff --git a/tests/cassettes/TestReActAgent.test_multi_step[False].yaml b/tests/cassettes/TestReActAgent.test_multi_step[False].yaml index 22141319..90244c84 100644 --- a/tests/cassettes/TestReActAgent.test_multi_step[False].yaml +++ b/tests/cassettes/TestReActAgent.test_multi_step[False].yaml @@ -1,19 +1,25 @@ interactions: - request: body: - '{"messages":[{"role":"system","content":"Answer the following questions + '{"messages": [{"role": "system", "content": "Answer the following questions as best you can, using the provided tools.\n\nUse the following format:\n\nThought: you should always think about what to do\nAction: the action to take, should be one of the provided tools with necessary arguments\nObservation: the result of the action\n... (this Thought/Action/Observation can repeat N times)\n\nExample:\n\nThought: I need to use the get_weather tool\nAction: get_weather(\"New York\", 7)\nObservation: - The 7 day forecast for New York is [...]"},{"role":"user","content":"Cast ''5.5'' - to a float, then to an integer, and finally use it to write a story of that - many words."}],"model":"gpt-4o-2024-08-06","n":1,"parallel_tool_calls":false,"stop":["Observation:","Action:"],"tool_choice":"none","tools":[{"type":"function","function":{"name":"print_story","description":"Print - a story.","parameters":{"type":"object","properties":{"story":{"description":"Story - to print.","title":"Story","type":"string"}},"required":["story"]}}},{"type":"function","function":{"name":"cast_float","description":"Cast - the input argument x to a float.","parameters":{"type":"object","properties":{"x":{"title":"X","type":"string"}},"required":["x"]}}},{"type":"function","function":{"name":"cast_int","description":"Cast - the input argument x to an integer.","parameters":{"type":"object","properties":{"x":{"title":"X","type":"number"}},"required":["x"]}}}]}' + The 7 day forecast for New York is [...]"}, {"role": "user", "content": "Cast + ''5.5'' to a float, then to an integer, and finally use it to write a story + of that many words."}], "model": "gpt-4o-mini-2024-07-18", "n": 1, "parallel_tool_calls": + false, "stop": ["Observation:", "Action:"], "tool_choice": "none", "tools": + [{"type": "function", "function": {"name": "print_story", "description": "Print + a story.", "parameters": {"type": "object", "properties": {"story": {"description": + "Story to print.", "title": "Story", "type": "string"}}, "required": ["story"]}}}, + {"type": "function", "function": {"name": "cast_float", "description": "Cast + the input argument x to a float.", "parameters": {"type": "object", "properties": + {"x": {"title": "X", "type": "string"}}, "required": ["x"]}}}, {"type": "function", + "function": {"name": "cast_int", "description": "Cast the input argument x to + an integer.", "parameters": {"type": "object", "properties": {"x": {"title": + "X", "type": "number"}}, "required": ["x"]}}}]}' headers: accept: - application/json @@ -22,13 +28,13 @@ interactions: connection: - keep-alive content-length: - - "1428" + - "1509" content-type: - application/json host: - api.openai.com user-agent: - - AsyncOpenAI/Python 1.56.2 + - AsyncOpenAI/Python 1.59.3 x-stainless-arch: - arm64 x-stainless-async: @@ -38,7 +44,7 @@ interactions: x-stainless-os: - MacOS x-stainless-package-version: - - 1.56.2 + - 1.59.3 x-stainless-raw-response: - "true" x-stainless-retry-count: @@ -52,19 +58,19 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAAAwAAAP//jJI/b9swEMV3fYoDlyySodhWYmnLkABFhwJNprSFQJEniQ1FsuQJ+Qd/ - 94KyYyloCmThcL97D+8d+JoAMCVZBUz0nMTgdHYl8+/9n+uX+5u20Q/6+rb52n3b9vePz0+7S5ZG - hW1+o6A31UrYwWkkZc0BC4+cMLqeX242u6LcXZQTGKxEHWWdo2xrs3W+3mb5LssvjsLeKoGBVfAj - AQB4nd4Y0Uh8YhXk6dtkwBB4h6w6LQEwb3WcMB6CCsQNsXSGwhpCM6W+6+3Y9VTBjfKBUvgCBlEC - WRA8EFCPEMgr08FZsSrOIuDQastp9dMsPT22Y+Cxkhm1Ps73p5Dads7bJhz5ad4qo0Jfe+TBmhgo - kHVsovsE4Nd0jPFdP+a8HRzVZB/QRMP1pjj4sfn8Mz0vj5Ascb1QFdv0A79aInGlw+KcTHDRo5yl - 8+35KJVdgGTR+t80H3kfmivTfcZ+BkKgI5S18yiVeN94XvMYf+f/1k5XngKz8BwIh7pVpkPvvDp8 - kNbVpSxyIcsyb1iyT/4CAAD//wMAzORJUikDAAA= + H4sIAAAAAAAAAwAAAP//jJI/b9swEMV3fYoDlyyWYUuRrWhLhqIFOhlZ+g8CTZ4kphRJkyejbeDv + XlB2LAVJgS4c7nfv4d3xnhMApiSrgImOk+idTu9dezzs3PHxsOs3xZ/PuH24Oz58+Mh3X74e2CIq + 7P4JBb2olsL2TiMpa85YeOSE0XW9zTdlma+zbAS9laijrHWU3tq0V0al2Sq7TVfbdF1e1J1VAgOr + 4FsCAPA8vjGnkfiLVbBavFR6DIG3yKprEwDzVscK4yGoQNwQW0xQWENoxuiPnR3ajir4BAZRAlkQ + PBBQhxDIK9PCTbEsbiLg0GjLCRrlAy2/m7mlx2YIPI5lBq0v9dM1o7at83YfLvxab5RRoas98mBN + zBPIOjbSUwLwY9zF8Go85rztHdVkf6KJhllenP3Y9AUTXZcXSJa4nqmKfPGOXy2RuNJhtk0muOhQ + TtJp9XyQys5AMpv6bZr3vM+TK9P+j/0EhEBHKGvnUSrxeuKpzWO80H+1Xbc8BmYB/VEJrEmhjz8h + seGDPt8NC78DYV83yrTonVfn42lcvZdlnmd3zSZnySn5CwAA//8DAJ2s0M9KAwAA headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8effbad9fb7067c4-SJC + - 90200cbb9efd15aa-SJC Connection: - keep-alive Content-Encoding: @@ -72,7 +78,7 @@ interactions: Content-Type: - application/json Date: - - Tue, 10 Dec 2024 19:44:30 GMT + - Tue, 14 Jan 2025 19:32:03 GMT Server: - cloudflare Transfer-Encoding: @@ -86,45 +92,52 @@ interactions: openai-organization: - future-house-xr4tdh openai-processing-ms: - - "551" + - "459" openai-version: - "2020-10-01" strict-transport-security: - max-age=31536000; includeSubDomains; preload x-ratelimit-limit-requests: - - "10000" + - "30000" x-ratelimit-limit-tokens: - - "30000000" + - "150000000" x-ratelimit-remaining-requests: - - "9999" + - "29999" x-ratelimit-remaining-tokens: - - "29999837" + - "149999837" x-ratelimit-reset-requests: - - 6ms + - 2ms x-ratelimit-reset-tokens: - 0s x-request-id: - - req_f77c374c33390f9d18b0a5f3b86ac75f + - req_13bac26d5a3880e5c6349b6c09a79084 status: code: 200 message: OK - request: body: - '{"messages":[{"role":"system","content":"Answer the following questions + '{"messages": [{"role": "system", "content": "Answer the following questions as best you can, using the provided tools.\n\nUse the following format:\n\nThought: you should always think about what to do\nAction: the action to take, should be one of the provided tools with necessary arguments\nObservation: the result of the action\n... (this Thought/Action/Observation can repeat N times)\n\nExample:\n\nThought: I need to use the get_weather tool\nAction: get_weather(\"New York\", 7)\nObservation: - The 7 day forecast for New York is [...]"},{"role":"user","content":"Cast ''5.5'' - to a float, then to an integer, and finally use it to write a story of that - many words."},{"role":"assistant","content":"Thought: First, I need to cast - the string ''5.5'' to a float.\n. Based on this reasoning, let''s select the - appropriate tool!\nAction: "},{"role":"user","content":"Continue..."}],"model":"gpt-4o-2024-08-06","n":1,"parallel_tool_calls":false,"stop":["Observation:","Action:"],"tool_choice":"required","tools":[{"type":"function","function":{"name":"print_story","description":"Print - a story.","parameters":{"type":"object","properties":{"story":{"description":"Story - to print.","title":"Story","type":"string"}},"required":["story"]}}},{"type":"function","function":{"name":"cast_float","description":"Cast - the input argument x to a float.","parameters":{"type":"object","properties":{"x":{"title":"X","type":"string"}},"required":["x"]}}},{"type":"function","function":{"name":"cast_int","description":"Cast - the input argument x to an integer.","parameters":{"type":"object","properties":{"x":{"title":"X","type":"number"}},"required":["x"]}}}]}' + The 7 day forecast for New York is [...]"}, {"role": "user", "content": "Cast + ''5.5'' to a float, then to an integer, and finally use it to write a story + of that many words."}, {"role": "assistant", "content": "Thought: I need to + cast the string ''5.5'' to a float first.\n. Based on this reasoning, let''s + select the appropriate tool!\nAction: "}, {"role": "user", "content": "Continue..."}], + "model": "gpt-4o-mini-2024-07-18", "n": 1, "parallel_tool_calls": false, "stop": + ["Observation:", "Action:"], "tool_choice": "required", "tools": [{"type": "function", + "function": {"name": "print_story", "description": "Print a story.", "parameters": + {"type": "object", "properties": {"story": {"description": "Story to print.", + "title": "Story", "type": "string"}}, "required": ["story"]}}}, {"type": "function", + "function": {"name": "cast_float", "description": "Cast the input argument x + to a float.", "parameters": {"type": "object", "properties": {"x": {"title": + "X", "type": "string"}}, "required": ["x"]}}}, {"type": "function", "function": + {"name": "cast_int", "description": "Cast the input argument x to an integer.", + "parameters": {"type": "object", "properties": {"x": {"title": "X", "type": + "number"}}, "required": ["x"]}}}]}' headers: accept: - application/json @@ -133,13 +146,13 @@ interactions: connection: - keep-alive content-length: - - "1638" + - "1726" content-type: - application/json host: - api.openai.com user-agent: - - AsyncOpenAI/Python 1.56.2 + - AsyncOpenAI/Python 1.59.3 x-stainless-arch: - arm64 x-stainless-async: @@ -149,7 +162,7 @@ interactions: x-stainless-os: - MacOS x-stainless-package-version: - - 1.56.2 + - 1.59.3 x-stainless-raw-response: - "true" x-stainless-retry-count: @@ -163,20 +176,20 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAAAwAAAP//jFPbitswEH33V4h5ThavE+fity29Qktp2bK0zWJkaexoVzckuSSE/Pti - O2s7aQr1gxBz5pyZORofIkJAcMgIsC0NTFk5vePxd7F79/HL++Jhlyy1Kn6YZPb100+bLp5g0jBM - 8YQsvLJumFFWYhBGdzBzSAM2qrfL2WyVrlfLuAWU4SgbWmXDdG6mSZzMp/FqGi9OxK0RDD1k5HdE - CCGH9mxa1Bx3kJFWpo0o9J5WCFmfRAg4I5sIUO+FD1QHmAwgMzqgbrrWtZQjIBgjc0alHAp332F0 - H3yiUubP8/rNZ8OK2f5Pcn+v+UP69pv6pT6M6nXSe9s2VNaa9f6M8D6eXRQjBDRV2BX0IS+loeGC - TQhQV9UKdWg6h8MGdhvINpDepBs4wlnyMbp2fxzZ4LCsPZUnf07xY2+4NJV1pvAX/kEptPDb3CH1 - 7RxjO6PXam0dqM9eDKwzyoY8mGfUjWyyXHWqMCzUgN4uTmAwgcoRaz2fXNHLOQYq2iftt4hRtkU+ - UIdtojUXZgREo9n/7uaadje/0NX/yA8AY2gD8tw65IKdTzykOWz+t3+l9S63DYPf+4AqL4Wu0Fkn - 2pWH0uZrnsaMr9dxAdExegEAAP//AwB1X7QN+wMAAA== + H4sIAAAAAAAAA4xTTW/bMAy9+1cIPMdFEuervnUd0HYFNuywYsMyGIpMO1plSZHorFmQ/z7YTmwn + y4D5YAh8fI/kE7UPGAOZQsxArDmJwqrwzuZb9+1+8X71abN+9ySmL48vjzT5jLm9e4FBxTCrnyjo + xLoRprAKSRrdwMIhJ6xUR/NotlhEo3FUA4VJUVW03FI4MWEhtQzHw/EkHM7D0eLIXhsp0EPMvgeM + Mbav/1WfOsU3iNlwcIoU6D3PEeI2iTFwRlUR4N5LT1wTDDpQGE2oq9Z1qVQPIGNUIrhSXeHm2/fO + nVlcqWS3edg+b2bPX1cP29fJhy+/nza/Pubz+169Rnpn64ayUovWpB7exuOLYoyB5gU2BT0lmTKc + LtiMAXd5WaCmqnPYL+FtCfESpjfTJRzgLPkQXDv/6NngMCs9V0d/jvFDa7gyuXVm5S/8g0xq6deJ + Q+7rOfp2BqdqdR0oz24MrDOFpYTMK+pKdjyfN6rQbVWHjmZHkAxx1WPdRoMrekmKxGV9pe0WCS7W + mHbUbpt4mUrTA4Le7H93c027mV/q/H/kO0AItIRpYh2mUpxP3KU5rB7dv9Jal+uGwaPbSoEJSXTV + faSY8VI1ywN+5wmLJJM6R2edrN8DZDZZpYsoGt9mswiCQ/AHAAD//wMAeKk/wB0EAAA= headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8effbade2f5e67c4-SJC + - 90200cbfcabf15aa-SJC Connection: - keep-alive Content-Encoding: @@ -184,7 +197,7 @@ interactions: Content-Type: - application/json Date: - - Tue, 10 Dec 2024 19:44:31 GMT + - Tue, 14 Jan 2025 19:32:03 GMT Server: - cloudflare Transfer-Encoding: @@ -198,46 +211,56 @@ interactions: openai-organization: - future-house-xr4tdh openai-processing-ms: - - "510" + - "509" openai-version: - "2020-10-01" strict-transport-security: - max-age=31536000; includeSubDomains; preload x-ratelimit-limit-requests: - - "10000" + - "30000" x-ratelimit-limit-tokens: - - "30000000" + - "150000000" x-ratelimit-remaining-requests: - - "9999" + - "29999" x-ratelimit-remaining-tokens: - - "29999800" + - "149999799" x-ratelimit-reset-requests: - - 6ms + - 2ms x-ratelimit-reset-tokens: - 0s x-request-id: - - req_a22592e960787b9681a567be8f799fb9 + - req_f4295206ce01a39595ce35aa8d2e86a6 status: code: 200 message: OK - request: body: - '{"messages":[{"role":"system","content":"Answer the following questions + '{"messages": [{"role": "system", "content": "Answer the following questions as best you can, using the provided tools.\n\nUse the following format:\n\nThought: you should always think about what to do\nAction: the action to take, should be one of the provided tools with necessary arguments\nObservation: the result of the action\n... (this Thought/Action/Observation can repeat N times)\n\nExample:\n\nThought: I need to use the get_weather tool\nAction: get_weather(\"New York\", 7)\nObservation: - The 7 day forecast for New York is [...]"},{"role":"user","content":"Cast ''5.5'' - to a float, then to an integer, and finally use it to write a story of that - many words."},{"role":"assistant","content":"Thought: First, I need to cast - the string ''5.5'' to a float.\n. Based on this reasoning, let''s select the - appropriate tool!\nAction: "},{"role":"user","content":"Continue..."},{"role":"assistant","content":null,"function_call":null,"tool_calls":[{"id":"call_k4uBLocb3yv2TTndW5DQmZmG","type":"function","function":{"arguments":"{\"x\": - \"5.5\"}","name":"cast_float"}}]},{"role":"tool","content":"Observation: 5.5","name":"cast_float","tool_call_id":"call_k4uBLocb3yv2TTndW5DQmZmG"}],"model":"gpt-4o-2024-08-06","n":1,"parallel_tool_calls":false,"stop":["Observation:","Action:"],"tool_choice":"none","tools":[{"type":"function","function":{"name":"print_story","description":"Print - a story.","parameters":{"type":"object","properties":{"story":{"description":"Story - to print.","title":"Story","type":"string"}},"required":["story"]}}},{"type":"function","function":{"name":"cast_float","description":"Cast - the input argument x to a float.","parameters":{"type":"object","properties":{"x":{"title":"X","type":"string"}},"required":["x"]}}},{"type":"function","function":{"name":"cast_int","description":"Cast - the input argument x to an integer.","parameters":{"type":"object","properties":{"x":{"title":"X","type":"number"}},"required":["x"]}}}]}' + The 7 day forecast for New York is [...]"}, {"role": "user", "content": "Cast + ''5.5'' to a float, then to an integer, and finally use it to write a story + of that many words."}, {"role": "assistant", "content": "Thought: I need to + cast the string ''5.5'' to a float first.\n. Based on this reasoning, let''s + select the appropriate tool!\nAction: "}, {"role": "user", "content": "Continue..."}, + {"role": "assistant", "content": null, "function_call": null, "tool_calls": + [{"id": "call_yqGvKq6KXbGvk4JUzIqwNg7C", "type": "function", "function": {"arguments": + "{\"x\": \"5.5\"}", "name": "cast_float"}}]}, {"role": "tool", "content": "Observation: + 5.5", "name": "cast_float", "tool_call_id": "call_yqGvKq6KXbGvk4JUzIqwNg7C"}], + "model": "gpt-4o-mini-2024-07-18", "n": 1, "parallel_tool_calls": false, "stop": + ["Observation:", "Action:"], "tool_choice": "none", "tools": [{"type": "function", + "function": {"name": "print_story", "description": "Print a story.", "parameters": + {"type": "object", "properties": {"story": {"description": "Story to print.", + "title": "Story", "type": "string"}}, "required": ["story"]}}}, {"type": "function", + "function": {"name": "cast_float", "description": "Cast the input argument x + to a float.", "parameters": {"type": "object", "properties": {"x": {"title": + "X", "type": "string"}}, "required": ["x"]}}}, {"type": "function", "function": + {"name": "cast_int", "description": "Cast the input argument x to an integer.", + "parameters": {"type": "object", "properties": {"x": {"title": "X", "type": + "number"}}, "required": ["x"]}}}]}' headers: accept: - application/json @@ -246,13 +269,13 @@ interactions: connection: - keep-alive content-length: - - "1939" + - "2051" content-type: - application/json host: - api.openai.com user-agent: - - AsyncOpenAI/Python 1.56.2 + - AsyncOpenAI/Python 1.59.3 x-stainless-arch: - arm64 x-stainless-async: @@ -262,7 +285,7 @@ interactions: x-stainless-os: - MacOS x-stainless-package-version: - - 1.56.2 + - 1.59.3 x-stainless-raw-response: - "true" x-stainless-retry-count: @@ -276,19 +299,19 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAAAwAAAP//jJJBb9swDIXv/hWELr3YgZvEbZLbtl4CBMM29DKsg6FItKVMlgSJ3lYU - +e+DnDR2sA7YRYf3kQ98pF4yAKYl2wATipPovCneyfLL4cPdp1btHrdCrb5aH3YPu88Ph/dzyfLU - 4fYHFPTaNROu8wZJO3vCIiAnTK6394vFqlqv7m8H0DmJJrW1noqlK+blfFmUq6K8OzcqpwVGtoFv - GQDAy/CmEa3E32wDZf6qdBgjb5FtLkUALDiTFMZj1JG4JZaPUDhLaIepH5XrW0Ub+Oh+ASlOsAXF - fyLEXgiMsemNeQbBI8FNNatugByQQmiM4wTVrMphCxZRJjCUXdGkcgvaErYYZk92OkbApo88bcH2 - xpz14yWXca0Pbh/P/KI32uqo6oA8OpsyRHKeDfSYAXwf9tdfrYT54DpPNbkfaJPholyf/Nh4sQmt - zpAccTPRl8v8Db9aInFt4uQCTHChUI6t47l4L7WbgGyS+u9p3vI+Jde2/R/7EQiBnlDWPqDU4jrx - WBYwfeh/lV22PAzM4nMk7OpG2xaDD/r0pxpfr2VVCrlel3uWHbM/AAAA//8DAJ7rg2RcAwAA + H4sIAAAAAAAAAwAAAP//jFLBitswEL37KwadthAHx052Xd/2uIcGFkJ7aItRpLGtrSwJaZxuWfLv + RXY2dmgLvRgzb97TezPzlgAwJVkFTHScRO90+ujak1df9ln5+pk+PR/2j9vdXuTD8wF1zlaRYY8v + KOidtRa2dxpJWTPBwiMnjKqbh+K+LItNXoxAbyXqSGsdpVub9sqoNM/ybZo9pJvywu6sEhhYBV8T + AIC38Rt9GomvrIJs9V7pMQTeIquuTQDMWx0rjIegAnFDbDWDwhpCM1o/dHZoO6pgb38CdZzgCTp+ + QqAOodGWE9zt1rsPK3gCgyiBLAgeCBTFX25AGcIW/fqbWT7hsRkCjzHNoPWlfr561rZ13h7DBb/W + G2VU6GqPPFgT/QWyjo3oOQH4Ps5muInLnLe9o5rsDzRRsMjKSY/NK5nRPL+AZInrBau4TPRWr5ZI + XOmwmC4TXHQoZ+q8Cj5IZRdAskj9p5u/aU/JlWn/R34GhEBHKGvnUSpxm3hu8xgv9l9t1ymPhllA + f1ICa1Lo4yYkNnzQ0x2x8CsQ9nWjTIveeTUdU+PqoyyLIv/Y3BcsOSe/AQAA//8DAHnpwXxaAwAA headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8effbae21a9367c4-SJC + - 90200cc3dd8e15aa-SJC Connection: - keep-alive Content-Encoding: @@ -296,7 +319,7 @@ interactions: Content-Type: - application/json Date: - - Tue, 10 Dec 2024 19:44:32 GMT + - Tue, 14 Jan 2025 19:32:04 GMT Server: - cloudflare Transfer-Encoding: @@ -310,49 +333,59 @@ interactions: openai-organization: - future-house-xr4tdh openai-processing-ms: - - "1085" + - "447" openai-version: - "2020-10-01" strict-transport-security: - max-age=31536000; includeSubDomains; preload x-ratelimit-limit-requests: - - "10000" + - "30000" x-ratelimit-limit-tokens: - - "30000000" + - "150000000" x-ratelimit-remaining-requests: - - "9999" + - "29999" x-ratelimit-remaining-tokens: - - "29999793" + - "149999793" x-ratelimit-reset-requests: - - 6ms + - 2ms x-ratelimit-reset-tokens: - 0s x-request-id: - - req_a936bbebea97c7fc58839d6d0f1e735d + - req_fadf7e1be52669ecf7f4684e067a2a79 status: code: 200 message: OK - request: body: - '{"messages":[{"role":"system","content":"Answer the following questions + '{"messages": [{"role": "system", "content": "Answer the following questions as best you can, using the provided tools.\n\nUse the following format:\n\nThought: you should always think about what to do\nAction: the action to take, should be one of the provided tools with necessary arguments\nObservation: the result of the action\n... (this Thought/Action/Observation can repeat N times)\n\nExample:\n\nThought: I need to use the get_weather tool\nAction: get_weather(\"New York\", 7)\nObservation: - The 7 day forecast for New York is [...]"},{"role":"user","content":"Cast ''5.5'' - to a float, then to an integer, and finally use it to write a story of that - many words."},{"role":"assistant","content":"Thought: First, I need to cast - the string ''5.5'' to a float.\n. Based on this reasoning, let''s select the - appropriate tool!\nAction: "},{"role":"user","content":"Continue..."},{"role":"assistant","content":null,"function_call":null,"tool_calls":[{"id":"call_k4uBLocb3yv2TTndW5DQmZmG","type":"function","function":{"arguments":"{\"x\": - \"5.5\"}","name":"cast_float"}}]},{"role":"tool","content":"Observation: 5.5","name":"cast_float","tool_call_id":"call_k4uBLocb3yv2TTndW5DQmZmG"},{"role":"assistant","content":"Thought: - Now that I have successfully cast ''5.5'' to the float 5.5, I need to cast the - float 5.5 to an integer.\n. Based on this reasoning, let''s select the appropriate - tool!\nAction: "},{"role":"user","content":"Continue..."}],"model":"gpt-4o-2024-08-06","n":1,"parallel_tool_calls":false,"stop":["Observation:","Action:"],"tool_choice":"required","tools":[{"type":"function","function":{"name":"print_story","description":"Print - a story.","parameters":{"type":"object","properties":{"story":{"description":"Story - to print.","title":"Story","type":"string"}},"required":["story"]}}},{"type":"function","function":{"name":"cast_float","description":"Cast - the input argument x to a float.","parameters":{"type":"object","properties":{"x":{"title":"X","type":"string"}},"required":["x"]}}},{"type":"function","function":{"name":"cast_int","description":"Cast - the input argument x to an integer.","parameters":{"type":"object","properties":{"x":{"title":"X","type":"number"}},"required":["x"]}}}]}' + The 7 day forecast for New York is [...]"}, {"role": "user", "content": "Cast + ''5.5'' to a float, then to an integer, and finally use it to write a story + of that many words."}, {"role": "assistant", "content": "Thought: I need to + cast the string ''5.5'' to a float first.\n. Based on this reasoning, let''s + select the appropriate tool!\nAction: "}, {"role": "user", "content": "Continue..."}, + {"role": "assistant", "content": null, "function_call": null, "tool_calls": + [{"id": "call_yqGvKq6KXbGvk4JUzIqwNg7C", "type": "function", "function": {"arguments": + "{\"x\": \"5.5\"}", "name": "cast_float"}}]}, {"role": "tool", "content": "Observation: + 5.5", "name": "cast_float", "tool_call_id": "call_yqGvKq6KXbGvk4JUzIqwNg7C"}, + {"role": "assistant", "content": "Thought: Now that I have the float (5.5), + I need to cast it to an integer.\n. Based on this reasoning, let''s select the + appropriate tool!\nAction: "}, {"role": "user", "content": "Continue..."}], + "model": "gpt-4o-mini-2024-07-18", "n": 1, "parallel_tool_calls": false, "stop": + ["Observation:", "Action:"], "tool_choice": "required", "tools": [{"type": "function", + "function": {"name": "print_story", "description": "Print a story.", "parameters": + {"type": "object", "properties": {"story": {"description": "Story to print.", + "title": "Story", "type": "string"}}, "required": ["story"]}}}, {"type": "function", + "function": {"name": "cast_float", "description": "Cast the input argument x + to a float.", "parameters": {"type": "object", "properties": {"x": {"title": + "X", "type": "string"}}, "required": ["x"]}}}, {"type": "function", "function": + {"name": "cast_int", "description": "Cast the input argument x to an integer.", + "parameters": {"type": "object", "properties": {"x": {"title": "X", "type": + "number"}}, "required": ["x"]}}}]}' headers: accept: - application/json @@ -361,13 +394,13 @@ interactions: connection: - keep-alive content-length: - - "2200" + - "2284" content-type: - application/json host: - api.openai.com user-agent: - - AsyncOpenAI/Python 1.56.2 + - AsyncOpenAI/Python 1.59.3 x-stainless-arch: - arm64 x-stainless-async: @@ -377,7 +410,7 @@ interactions: x-stainless-os: - MacOS x-stainless-package-version: - - 1.56.2 + - 1.59.3 x-stainless-raw-response: - "true" x-stainless-retry-count: @@ -391,20 +424,20 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAAAwAAAP//jFNNi9swEL37V4g5x4s3cRLHty0tpZCWsqWnbjGKNHbU1VclmU0a8t+L - P2I72yzUByHmzbw38zQ+RYSA4JATYHsamLIyfuDJ4/M38cfL9Qe33R75IavM55f0++GQfoJZU2F2 - v5CFS9UdM8pKDMLoDmYOacCG9X69WGTLTbaet4AyHGVTVtkQpyaeJ/M0TrI4WfWFeyMYesjJj4gQ - Qk7t2bSoOR4gJ8nsElHoPa0Q8iGJEHBGNhGg3gsfqA4wG0FmdEDddK1rKSdAMEYWjEo5CnffaXIf - faJSFunaqq/i99a8Xz2+kzv1or5kWfWxnuh11EfbNlTWmg3+TPAhnr8SIwQ0VdgJ+lCIq1n6DOqq - WqEOTd9weoLDE+TLu+UZrhLP0a37z4kBDsvaU9k708fPg9XSVNaZnX/lHJRCC78vHFLfTjA1Mrqo - tTpQX70VWGeUDUUwz6gb2sUq61hhXKURvV/1YDCByklVls5u8BUcAxXtYw77wyjbIx9Lxz2iNRdm - AkST2f/t5hZ3N7/Q1f/QjwBjaAPywjrkgl1PPKY5bP60t9IGl9uGwR99QFWUQlforBPtskNpiw1f - JoxvNskOonP0FwAA//8DAFcMvhr1AwAA + H4sIAAAAAAAAA4xTyW7bMBC96yuIOVuBJXmrbkUL2F1yKFI3KJpCoKmRxJQiCZISEhj+90CLtaQp + UB0EYt7MezOPw7NHCPAUYgKsoI6VWvjvdV7b77eh+bn++mFbl+y2Unf1Pjgcjt9qWDQV6vSIzF2r + bpgqtUDHlexgZpA6bFiDbbTZ7aIgXLVAqVIUTVmunb9Sfskl98NluPKXWz/Y9dWF4gwtxOSXRwgh + 5/bf9ClTfIKYLBfXSInW0hwhHpIIAaNEEwFqLbeOSgeLEWRKOpRN67ISYgI4pUTCqBCjcPedJ+fR + LCpEsj991tFydffpeLjfu+JLTe8ffwQfjxO9jvpZtw1llWSDSRN8iMevxAgBSUvsBK1L+GyWPoOa + vCpRuqZvOD/A0wPE65v1BWaJF++t8++JAQazylLRO9PHL4PVQuXaqJN95RxkXHJbJAapbSeYGuld + 1VodqGZ3BdqoUrvEqT8oG9povepYYdynEQ02PeiUo2JSte3XYc6XpOgoby9z2B9GWYHpWDruEa1S + riaAN5n9727e4u7m5zL/H/oRYAy1wzTRBlPO5hOPaQab5/avtMHltmGwaGrOMHEcTXMfKWa0Et3i + gH22Dssk4zJHow1vXwJkOjmluygK32WbCLyL9wIAAP//AwBJRNBbFwQAAA== headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8effbae9f94c67c4-SJC + - 90200cc8391315aa-SJC Connection: - keep-alive Content-Encoding: @@ -412,7 +445,7 @@ interactions: Content-Type: - application/json Date: - - Tue, 10 Dec 2024 19:44:33 GMT + - Tue, 14 Jan 2025 19:32:04 GMT Server: - cloudflare Transfer-Encoding: @@ -426,50 +459,62 @@ interactions: openai-organization: - future-house-xr4tdh openai-processing-ms: - - "888" + - "356" openai-version: - "2020-10-01" strict-transport-security: - max-age=31536000; includeSubDomains; preload x-ratelimit-limit-requests: - - "10000" + - "30000" x-ratelimit-limit-tokens: - - "30000000" + - "150000000" x-ratelimit-remaining-requests: - - "9999" + - "29999" x-ratelimit-remaining-tokens: - - "29999744" + - "149999753" x-ratelimit-reset-requests: - - 6ms + - 2ms x-ratelimit-reset-tokens: - 0s x-request-id: - - req_9f0fb599bd9aa3cbc99412fbeee4d0c8 + - req_e663f80b9fc07001b5a30a097fb4027d status: code: 200 message: OK - request: body: - '{"messages":[{"role":"system","content":"Answer the following questions + '{"messages": [{"role": "system", "content": "Answer the following questions as best you can, using the provided tools.\n\nUse the following format:\n\nThought: you should always think about what to do\nAction: the action to take, should be one of the provided tools with necessary arguments\nObservation: the result of the action\n... (this Thought/Action/Observation can repeat N times)\n\nExample:\n\nThought: I need to use the get_weather tool\nAction: get_weather(\"New York\", 7)\nObservation: - The 7 day forecast for New York is [...]"},{"role":"user","content":"Cast ''5.5'' - to a float, then to an integer, and finally use it to write a story of that - many words."},{"role":"assistant","content":"Thought: First, I need to cast - the string ''5.5'' to a float.\n. Based on this reasoning, let''s select the - appropriate tool!\nAction: "},{"role":"user","content":"Continue..."},{"role":"assistant","content":null,"function_call":null,"tool_calls":[{"id":"call_k4uBLocb3yv2TTndW5DQmZmG","type":"function","function":{"arguments":"{\"x\": - \"5.5\"}","name":"cast_float"}}]},{"role":"tool","content":"Observation: 5.5","name":"cast_float","tool_call_id":"call_k4uBLocb3yv2TTndW5DQmZmG"},{"role":"assistant","content":"Thought: - Now that I have successfully cast ''5.5'' to the float 5.5, I need to cast the - float 5.5 to an integer.\n. Based on this reasoning, let''s select the appropriate - tool!\nAction: "},{"role":"user","content":"Continue..."},{"role":"assistant","content":null,"function_call":null,"tool_calls":[{"id":"call_47pmPiqLoD6RBlbmwmN88gGu","type":"function","function":{"arguments":"{\"x\": - 5.5}","name":"cast_int"}}]},{"role":"tool","content":"Observation: 5","name":"cast_int","tool_call_id":"call_47pmPiqLoD6RBlbmwmN88gGu"}],"model":"gpt-4o-2024-08-06","n":1,"parallel_tool_calls":false,"stop":["Observation:","Action:"],"tool_choice":"none","tools":[{"type":"function","function":{"name":"print_story","description":"Print - a story.","parameters":{"type":"object","properties":{"story":{"description":"Story - to print.","title":"Story","type":"string"}},"required":["story"]}}},{"type":"function","function":{"name":"cast_float","description":"Cast - the input argument x to a float.","parameters":{"type":"object","properties":{"x":{"title":"X","type":"string"}},"required":["x"]}}},{"type":"function","function":{"name":"cast_int","description":"Cast - the input argument x to an integer.","parameters":{"type":"object","properties":{"x":{"title":"X","type":"number"}},"required":["x"]}}}]}' + The 7 day forecast for New York is [...]"}, {"role": "user", "content": "Cast + ''5.5'' to a float, then to an integer, and finally use it to write a story + of that many words."}, {"role": "assistant", "content": "Thought: I need to + cast the string ''5.5'' to a float first.\n. Based on this reasoning, let''s + select the appropriate tool!\nAction: "}, {"role": "user", "content": "Continue..."}, + {"role": "assistant", "content": null, "function_call": null, "tool_calls": + [{"id": "call_yqGvKq6KXbGvk4JUzIqwNg7C", "type": "function", "function": {"arguments": + "{\"x\": \"5.5\"}", "name": "cast_float"}}]}, {"role": "tool", "content": "Observation: + 5.5", "name": "cast_float", "tool_call_id": "call_yqGvKq6KXbGvk4JUzIqwNg7C"}, + {"role": "assistant", "content": "Thought: Now that I have the float (5.5), + I need to cast it to an integer.\n. Based on this reasoning, let''s select the + appropriate tool!\nAction: "}, {"role": "user", "content": "Continue..."}, {"role": + "assistant", "content": null, "function_call": null, "tool_calls": [{"id": "call_GbJp304SIUHWGthKvaWjV1DU", + "type": "function", "function": {"arguments": "{\"x\": 5.5}", "name": "cast_int"}}]}, + {"role": "tool", "content": "Observation: 5", "name": "cast_int", "tool_call_id": + "call_GbJp304SIUHWGthKvaWjV1DU"}], "model": "gpt-4o-mini-2024-07-18", "n": 1, + "parallel_tool_calls": false, "stop": ["Observation:", "Action:"], "tool_choice": + "none", "tools": [{"type": "function", "function": {"name": "print_story", "description": + "Print a story.", "parameters": {"type": "object", "properties": {"story": {"description": + "Story to print.", "title": "Story", "type": "string"}}, "required": ["story"]}}}, + {"type": "function", "function": {"name": "cast_float", "description": "Cast + the input argument x to a float.", "parameters": {"type": "object", "properties": + {"x": {"title": "X", "type": "string"}}, "required": ["x"]}}}, {"type": "function", + "function": {"name": "cast_int", "description": "Cast the input argument x to + an integer.", "parameters": {"type": "object", "properties": {"x": {"title": + "X", "type": "number"}}, "required": ["x"]}}}]}' headers: accept: - application/json @@ -478,13 +523,13 @@ interactions: connection: - keep-alive content-length: - - "2491" + - "2599" content-type: - application/json host: - api.openai.com user-agent: - - AsyncOpenAI/Python 1.56.2 + - AsyncOpenAI/Python 1.59.3 x-stainless-arch: - arm64 x-stainless-async: @@ -494,7 +539,7 @@ interactions: x-stainless-os: - MacOS x-stainless-package-version: - - 1.56.2 + - 1.59.3 x-stainless-raw-response: - "true" x-stainless-retry-count: @@ -508,19 +553,20 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAAAwAAAP//jFLBjtMwFLznK558blbZpqFNbwiJZUFaaRFwARS59mti6vhZ9st2q1X/ - HSXNNq0AiYsPM2/GM89+SQCE0WINQjWSVett+lZnn+37x9t3n4q7XXb34Wv39PAx5KGsH3ffxKxX - 0OYXKn5V3ShqvUU25E60CigZe9fbZZ6vinK1zAeiJY22l9We0wWl82y+SLNVmr0ZhQ0ZhVGs4XsC - APAynH1Ep/FZrCGbvSItxihrFOvzEIAIZHtEyBhNZOlYzCZSkWN0Q+ovDXV1w2t4oD1wIxnuoZFP - CNwgGMdYY4BiBvfgEDUwwT4YRpAQmcIBeitpnHE14LNUbA9QwJ6Cjjc/3OWdAbddlH1l11k74sdz - CUu1D7SJI3/Gt8aZ2FQBZSTXB45MXgzsMQH4OSyru+ovfKDWc8W0Q9cb5uXy5Cem55nYeT6STCzt - hC/m44qv/SqNLI2NF+sWSqoG9SSd3kZ22tAFkVy0/jPN37xPzY2r/8d+IpRCz6grH1Abdd14GgvY - /95/jZ23PAQW8RAZ22prXI3BB3P6QFtflbrIlC7LbCOSY/IbAAD//wMArvA5aEkDAAA= + H4sIAAAAAAAAAwAAAP//jFLBattAEL3rK4a95GIb27IS17dCD01JC4VQCk0R692RtO1qZ7szsmtC + /r1IdiyHptDLHt6b93jzZh8zAOWs2oAyjRbTRj99G+ud3Bm6/1rt3n14n7ovHxd3Vfz8q5pzoSa9 + grY/0MizamaojR7FUTjSJqEW7F0XN/n1ep0vlsVAtGTR97I6ynRF09YFN13Ol6vp/Ga6WJ/UDTmD + rDbwLQMAeBzePmew+FttYD55Rlpk1jWqzXkIQCXyPaI0s2PRQdRkJA0FwTBEv2+oqxvZwC00eofA + nTHIXHXeH8BoFrgqZsUVCIEO4IJgjWkC+8aZBhxDMYNPtIdbCIi2n9onJwgaWCgdgCooYE/J8uwh + XEZIWHWs+xpC5/0Jfzrv5KmOibZ84s945YLjpkyomUKfn4WiGtinDOD70F33og4VE7VRSqGfGHrD + fJ0f/dR4sgt2eSKFRPsRXy2KySt+pUXRzvNF+8po06AdpeOpdGcdXRDZxdZ/p3nN+7i5C/X/2I+E + MRgFbRkTWmdebjyOJex/9L/Gzi0PgRVj2jmDpThM/SUsVrrzx3+m+MCCbVm5UGOKyR0/WxXLrV3n + +fJNdZ2r7Cn7AwAA//8DAGZVwy96AwAA headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8effbaf05e8b67c4-SJC + - 90200ccc0be015aa-SJC Connection: - keep-alive Content-Encoding: @@ -528,7 +574,7 @@ interactions: Content-Type: - application/json Date: - - Tue, 10 Dec 2024 19:44:34 GMT + - Tue, 14 Jan 2025 19:32:05 GMT Server: - cloudflare Transfer-Encoding: @@ -542,53 +588,65 @@ interactions: openai-organization: - future-house-xr4tdh openai-processing-ms: - - "742" + - "603" openai-version: - "2020-10-01" strict-transport-security: - max-age=31536000; includeSubDomains; preload x-ratelimit-limit-requests: - - "10000" + - "30000" x-ratelimit-limit-tokens: - - "30000000" + - "150000000" x-ratelimit-remaining-requests: - - "9999" + - "29999" x-ratelimit-remaining-tokens: - - "29999739" + - "149999747" x-ratelimit-reset-requests: - - 6ms + - 2ms x-ratelimit-reset-tokens: - 0s x-request-id: - - req_647d24cea03da8cc711d8d751099c15a + - req_4e3773ea912b297b912c1cdb31c5dfe8 status: code: 200 message: OK - request: body: - '{"messages":[{"role":"system","content":"Answer the following questions + '{"messages": [{"role": "system", "content": "Answer the following questions as best you can, using the provided tools.\n\nUse the following format:\n\nThought: you should always think about what to do\nAction: the action to take, should be one of the provided tools with necessary arguments\nObservation: the result of the action\n... (this Thought/Action/Observation can repeat N times)\n\nExample:\n\nThought: I need to use the get_weather tool\nAction: get_weather(\"New York\", 7)\nObservation: - The 7 day forecast for New York is [...]"},{"role":"user","content":"Cast ''5.5'' - to a float, then to an integer, and finally use it to write a story of that - many words."},{"role":"assistant","content":"Thought: First, I need to cast - the string ''5.5'' to a float.\n. Based on this reasoning, let''s select the - appropriate tool!\nAction: "},{"role":"user","content":"Continue..."},{"role":"assistant","content":null,"function_call":null,"tool_calls":[{"id":"call_k4uBLocb3yv2TTndW5DQmZmG","type":"function","function":{"arguments":"{\"x\": - \"5.5\"}","name":"cast_float"}}]},{"role":"tool","content":"Observation: 5.5","name":"cast_float","tool_call_id":"call_k4uBLocb3yv2TTndW5DQmZmG"},{"role":"assistant","content":"Thought: - Now that I have successfully cast ''5.5'' to the float 5.5, I need to cast the - float 5.5 to an integer.\n. Based on this reasoning, let''s select the appropriate - tool!\nAction: "},{"role":"user","content":"Continue..."},{"role":"assistant","content":null,"function_call":null,"tool_calls":[{"id":"call_47pmPiqLoD6RBlbmwmN88gGu","type":"function","function":{"arguments":"{\"x\": - 5.5}","name":"cast_int"}}]},{"role":"tool","content":"Observation: 5","name":"cast_int","tool_call_id":"call_47pmPiqLoD6RBlbmwmN88gGu"},{"role":"assistant","content":"Thought: - Now that I have the integer 5, I need to write a story containing exactly 5 - words.\n. Based on this reasoning, let''s select the appropriate tool!\nAction: - "},{"role":"user","content":"Continue..."}],"model":"gpt-4o-2024-08-06","n":1,"parallel_tool_calls":false,"stop":["Observation:","Action:"],"tool_choice":"required","tools":[{"type":"function","function":{"name":"print_story","description":"Print - a story.","parameters":{"type":"object","properties":{"story":{"description":"Story - to print.","title":"Story","type":"string"}},"required":["story"]}}},{"type":"function","function":{"name":"cast_float","description":"Cast - the input argument x to a float.","parameters":{"type":"object","properties":{"x":{"title":"X","type":"string"}},"required":["x"]}}},{"type":"function","function":{"name":"cast_int","description":"Cast - the input argument x to an integer.","parameters":{"type":"object","properties":{"x":{"title":"X","type":"number"}},"required":["x"]}}}]}' + The 7 day forecast for New York is [...]"}, {"role": "user", "content": "Cast + ''5.5'' to a float, then to an integer, and finally use it to write a story + of that many words."}, {"role": "assistant", "content": "Thought: I need to + cast the string ''5.5'' to a float first.\n. Based on this reasoning, let''s + select the appropriate tool!\nAction: "}, {"role": "user", "content": "Continue..."}, + {"role": "assistant", "content": null, "function_call": null, "tool_calls": + [{"id": "call_yqGvKq6KXbGvk4JUzIqwNg7C", "type": "function", "function": {"arguments": + "{\"x\": \"5.5\"}", "name": "cast_float"}}]}, {"role": "tool", "content": "Observation: + 5.5", "name": "cast_float", "tool_call_id": "call_yqGvKq6KXbGvk4JUzIqwNg7C"}, + {"role": "assistant", "content": "Thought: Now that I have the float (5.5), + I need to cast it to an integer.\n. Based on this reasoning, let''s select the + appropriate tool!\nAction: "}, {"role": "user", "content": "Continue..."}, {"role": + "assistant", "content": null, "function_call": null, "tool_calls": [{"id": "call_GbJp304SIUHWGthKvaWjV1DU", + "type": "function", "function": {"arguments": "{\"x\": 5.5}", "name": "cast_int"}}]}, + {"role": "tool", "content": "Observation: 5", "name": "cast_int", "tool_call_id": + "call_GbJp304SIUHWGthKvaWjV1DU"}, {"role": "assistant", "content": "Thought: + I have successfully cast ''5.5'' to an integer, which is 5. Now I need to write + a story of 5 words.\n. Based on this reasoning, let''s select the appropriate + tool!\nAction: "}, {"role": "user", "content": "Continue..."}], "model": "gpt-4o-mini-2024-07-18", + "n": 1, "parallel_tool_calls": false, "stop": ["Observation:", "Action:"], "tool_choice": + "required", "tools": [{"type": "function", "function": {"name": "print_story", + "description": "Print a story.", "parameters": {"type": "object", "properties": + {"story": {"description": "Story to print.", "title": "Story", "type": "string"}}, + "required": ["story"]}}}, {"type": "function", "function": {"name": "cast_float", + "description": "Cast the input argument x to a float.", "parameters": {"type": + "object", "properties": {"x": {"title": "X", "type": "string"}}, "required": + ["x"]}}}, {"type": "function", "function": {"name": "cast_int", "description": + "Cast the input argument x to an integer.", "parameters": {"type": "object", + "properties": {"x": {"title": "X", "type": "number"}}, "required": ["x"]}}}]}' headers: accept: - application/json @@ -597,13 +655,13 @@ interactions: connection: - keep-alive content-length: - - "2733" + - "2864" content-type: - application/json host: - api.openai.com user-agent: - - AsyncOpenAI/Python 1.56.2 + - AsyncOpenAI/Python 1.59.3 x-stainless-arch: - arm64 x-stainless-async: @@ -613,7 +671,7 @@ interactions: x-stainless-os: - MacOS x-stainless-package-version: - - 1.56.2 + - 1.59.3 x-stainless-raw-response: - "true" x-stainless-retry-count: @@ -627,20 +685,21 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAAAwAAAP//jFNNi9swEL37V4g5J4ubePPhW9nuIZTdQFuW3TTFKNLY0VZfSDJsCPnv - xUpiO2kK9UGIefPezDyN9wkhIDjkBNiWBqasHH7m6Tf15D5Wj4uvi4e3xcp9efu+eX5+dPVyBoOG - YTbvyMKZdceMshKDMPoIM4c0YKP6aToez+7ns2kWAWU4yoZW2TDMzHCUjrJhOhumkxNxawRDDzn5 - mRBCyD6eTYua4wfkJB2cIwq9pxVC3iYRAs7IJgLUe+ED1QEGHciMDqibrnUtZQ8IxsiCUSm7wsdv - 37t3PlEpi6WZ7p4mD6P3lx91iQv3qsbl62r20qt3lN7Z2FBZa9b608PbeH5VjBDQVEWudUKHwgfj - dld0QoC6qlaoQ9M67NcQ09aQr2GpGZLaGk0oCULhgGyptUKj98ShqDTyuzUc4ELxkNy6/+qZ5bCs - PZUnF0/xQ/ss0lTWmY2/chlKoYXfFg6pj9P2TU/O1WIdqC/eFawzyoYimN+oG9ksy46q0K1dh45G - JzCYQGWPNZkMbugVHAMV8eHbXWOUbZF31G7naM2F6QFJb/a/u7mlfZxf6Op/5DuAMbQBeWEdcsEu - J+7SHDZ/5b/SWpdjw+B3PqAqSqErdHHF4p7aYs7vU8bn83QDySH5AwAA//8DAGfXRGQhBAAA + H4sIAAAAAAAAA4xTwY6bMBC98xXWnMMqATbLcovUS6VI26a7lapmhRwzgFtju7ZJm0b59wqTAEm3 + Ujkga57fm5k342NACPACMgKspo41WoQrXe3bd+X6pf45pyv1/CNhqf398bnlX5ICZh1D7b4hcxfW + HVONFui4kj3MDFKHneriIV6mabyIlh5oVIGio1XahYkKGy55GM2jJJw/hIv0zK4VZ2ghI18DQgg5 + +n9XpyzwF2RkPrtEGrSWVgjZcIkQMEp0EaDWcuuodDAbQaakQ9mVLlshJoBTSuSMCjEm7r/j5Dya + RYXI3z9t6mSzXm3qD2tGP31O0v1Liev7Sb5e+qB9QWUr2WDSBB/i2U0yQkDSxnO14dLl1ilzuKET + AtRUbYPSdaXDcQv+2hayLTxJhqTVShJKHG9wRmqqNZdo7d0WTnCldAreOr9OTDJYtpaKs3vn+GkY + h1CVNmpnb9yFkktu69wgtb7LqdnBJZvPA+3VPEEb1WiXO/UdZSebxI+9Kow7N6LReTPAKUfFhHV/ + YV3p5QU6yv3Ahx1jlNVYjNRx12hbcDUBgknvf1fzlnbfP5fV/8iPAGOoHRa5Nlhwdt3xeM1g9yT/ + dW1w2RcMFs2eM8wdR9PNo8CStqJ/KGAP1mGTl1xWaPze+eXV+a5I4zh6LJcxBKfgDwAAAP//AwBn + +DyhOwQAAA== headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8effbaf5daeb67c4-SJC + - 90200cd0cf9d15aa-SJC Connection: - keep-alive Content-Encoding: @@ -648,7 +707,7 @@ interactions: Content-Type: - application/json Date: - - Tue, 10 Dec 2024 19:44:35 GMT + - Tue, 14 Jan 2025 19:32:06 GMT Server: - cloudflare Transfer-Encoding: @@ -662,25 +721,25 @@ interactions: openai-organization: - future-house-xr4tdh openai-processing-ms: - - "690" + - "518" openai-version: - "2020-10-01" strict-transport-security: - max-age=31536000; includeSubDomains; preload x-ratelimit-limit-requests: - - "10000" + - "30000" x-ratelimit-limit-tokens: - - "30000000" + - "150000000" x-ratelimit-remaining-requests: - - "9999" + - "29999" x-ratelimit-remaining-tokens: - - "29999692" + - "149999699" x-ratelimit-reset-requests: - - 6ms + - 2ms x-ratelimit-reset-tokens: - 0s x-request-id: - - req_3dc231dbb4775d7cb77a7e910a5c5910 + - req_a1358af7d988cb6638e31e08e039518a status: code: 200 message: OK diff --git a/tests/cassettes/TestReActAgent.test_multi_step[True].yaml b/tests/cassettes/TestReActAgent.test_multi_step[True].yaml index d98bab28..249870fd 100644 --- a/tests/cassettes/TestReActAgent.test_multi_step[True].yaml +++ b/tests/cassettes/TestReActAgent.test_multi_step[True].yaml @@ -1,7 +1,7 @@ interactions: - request: body: - '{"messages":[{"role":"system","content":"Answer the following questions + '{"messages": [{"role": "system", "content": "Answer the following questions as best you can. You have access to the following tools:\n\nNAME: print_story\n\nSYNOPSIS:\n print_story(string story)\n\nDESCRIPTION:\n Print a story.\n\nPARAMETERS:\n story (string): Story to print.\n\nNAME: cast_float\n\nSYNOPSIS:\n cast_float(string x)\n\nDESCRIPTION:\n Cast @@ -14,9 +14,9 @@ interactions: tuple\nObservation: the result of the action\n... (this Thought/Action/Action Input/Observation can repeat N times)\n\nExample:\n\nThought: I need to use the get_weather tool\nAction: get_weather\nAction Input: \"New York\", 7\nObservation: - The 7 day forecast for New York is [...]"},{"role":"user","content":"Cast ''5.5'' - to a float, then to an integer, and finally use it to write a story of that - many words."}],"model":"gpt-4o-2024-08-06","n":1,"stop":["Observation:"]}' + The 7 day forecast for New York is [...]"}, {"role": "user", "content": "Cast + ''5.5'' to a float, then to an integer, and finally use it to write a story + of that many words."}], "model": "gpt-4o-mini-2024-07-18", "n": 1, "stop": ["Observation:"]}' headers: accept: - application/json @@ -25,13 +25,13 @@ interactions: connection: - keep-alive content-length: - - "1331" + - "1350" content-type: - application/json host: - api.openai.com user-agent: - - AsyncOpenAI/Python 1.56.2 + - AsyncOpenAI/Python 1.59.3 x-stainless-arch: - arm64 x-stainless-async: @@ -41,7 +41,7 @@ interactions: x-stainless-os: - MacOS x-stainless-package-version: - - 1.56.2 + - 1.59.3 x-stainless-raw-response: - "true" x-stainless-retry-count: @@ -55,19 +55,20 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAAA4xSTWvcMBS8+1c8dEkL9uLdjTe7vgVCIYdCUxba0C1GKz3bamVJlZ5pStj/XuT9 - sENTyEWHmTfDzHt6TgCYkqwEJlpOonM6u5X5ZyweHraP+y9f1fxJf3z88Kn9Je62v/0NS6PC7n+g - oLNqJmznNJKy5kgLj5wwus5vlst1sVmvVgPRWYk6yhpH2bXNFvniOsvXWb46CVurBAZWwrcEAOB5 - eGNEI/GJlZCnZ6TDEHiDrLwMATBvdUQYD0EF4oZYOpLCGkIzpN62tm9aKuEeDKIEslArHwgEDwTU - IgTyyjRwVcyKq0hzqLXlNNuZWxGLlsNoNYBnDO6N66mEdztWzIodS9/vzDSAx7oPPPY3vdYn/HBp - pG3jvN2HE3/Ba2VUaCuPPFgT0weyjg3sIQH4Pmyuf7EM5rztHFVkf6KJhov18ujHxluN7HJxIskS - 1xN8XqSv+FUSiSsdJrtngosW5SgdD8V7qeyESCat/03zmvexuTLNW+xHQgh0hLJyHqUSLxuPYx7j - V/7f2GXLQ2AW/gTCrqqVadA7r46/qXbVRha5kJtNvmfJIfkLAAD//wMA5RGCp1YDAAA= + H4sIAAAAAAAAAwAAAP//jFLBbtswDL37KwhdugFxEMdp4/lWbBhWoOtpu6QdDEWiba2ypEl0saHI + vw+yk9jBNmAXHd7je+Ij+ZoAMCVZCUy0nETndHrrmpdulz3Ihx8ft/j5vth9+hDwfrd5zt9/ZYuo + sPvvKOikWgrbOY2krBlp4ZETRtdsm98URZ5lxUB0VqKOssZRurFpp4xK16v1Jl1t06w4qlurBAZW + wmMCAPA6vLFPI/EnK2G1OCEdhsAbZOW5CIB5qyPCeAgqEDfEFhMprCE0Q+tfWts3LZVwBwZRAlmo + lQ8EggeCq+vl9VXEONTaclo+mVsRI5YDXw3gCYM743oq4c2gWrx9MvM/PdZ94DG36bU+4odzCG0b + 5+0+HPkzXiujQlt55MGa2HAg69jAHhKAb8Ow+ov8zHnbOarIPqOJhusiH/3YtKOJzY+TZGSJ6xme + nVQXfpVE4kqH2biZ4KJFOUmn3fBeKjsjklnqP7v5m/eYXJnmf+wnQgh0hLJyHqUSl4mnMo/xhP9V + dp7y0DAL6F+UwIoU+rgJiTXv9XhYLPwKhF1VK9Ogd16N11W7ai+LPF+/q29ylhyS3wAAAP//AwDb + pFDzawMAAA== headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8effbac5baf7d039-SJC + - 90200c9faf4967f0-SJC Connection: - keep-alive Content-Encoding: @@ -75,14 +76,14 @@ interactions: Content-Type: - application/json Date: - - Tue, 10 Dec 2024 19:44:27 GMT + - Tue, 14 Jan 2025 19:31:59 GMT Server: - cloudflare Set-Cookie: - - __cf_bm=r6dZddUDWfN_Os.WIvmSRteRx9TQDrAqu8ra.Qbl9Rg-1733859867-1.0.1.1-bHAbl3a2MCrOO2KjtKMQsO2qa26vpz8SS4okiAhw16opeBH13XEmOd8TzokuXcMbLWibxkSbI.69x8G.cQ2EHA; - path=/; expires=Tue, 10-Dec-24 20:14:27 GMT; domain=.api.openai.com; HttpOnly; + - __cf_bm=ZeAftDaaDTNgvEYxvrTS7Qn.2maKkMyMhCKhoIAwV6M-1736883119-1.0.1.1-DZsirhk2wmSnXtCkn3KSM00Z_Yv4IL.ZUdme1ZqoMs_fg8Hs8H0B.f.uYNctxohI7FgOxhpcUu08amjQS.4.bQ; + path=/; expires=Tue, 14-Jan-25 20:01:59 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None - - _cfuvid=gioa0R.ivKqZKaFec_dzeeGl_QNH0QCLOItZoOpXstA-1733859867523-0.0.1.1-604800000; + - _cfuvid=ROTfDjEa2oE7LIiChbxsn.iXGdnoSAyl_8_SbeWSI8w-1736883119068-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None Transfer-Encoding: - chunked @@ -95,31 +96,31 @@ interactions: openai-organization: - future-house-xr4tdh openai-processing-ms: - - "895" + - "936" openai-version: - "2020-10-01" strict-transport-security: - max-age=31536000; includeSubDomains; preload x-ratelimit-limit-requests: - - "10000" + - "30000" x-ratelimit-limit-tokens: - - "30000000" + - "150000000" x-ratelimit-remaining-requests: - - "9999" + - "29999" x-ratelimit-remaining-tokens: - - "29999693" + - "149999693" x-ratelimit-reset-requests: - - 6ms + - 2ms x-ratelimit-reset-tokens: - 0s x-request-id: - - req_5fc910e91f3fcf4bd0193ba919285ab0 + - req_542f08a4f5be8f4bbe577d26d43e73da status: code: 200 message: OK - request: body: - '{"messages":[{"role":"system","content":"Answer the following questions + '{"messages": [{"role": "system", "content": "Answer the following questions as best you can. You have access to the following tools:\n\nNAME: print_story\n\nSYNOPSIS:\n print_story(string story)\n\nDESCRIPTION:\n Print a story.\n\nPARAMETERS:\n story (string): Story to print.\n\nNAME: cast_float\n\nSYNOPSIS:\n cast_float(string x)\n\nDESCRIPTION:\n Cast @@ -132,11 +133,12 @@ interactions: tuple\nObservation: the result of the action\n... (this Thought/Action/Action Input/Observation can repeat N times)\n\nExample:\n\nThought: I need to use the get_weather tool\nAction: get_weather\nAction Input: \"New York\", 7\nObservation: - The 7 day forecast for New York is [...]"},{"role":"user","content":"Cast ''5.5'' - to a float, then to an integer, and finally use it to write a story of that - many words."},{"role":"assistant","content":"Thought: I need to first cast the - string ''5.5'' to a float.\nAction: cast_float\nAction Input: (\"5.5\",)"},{"role":"user","content":"Observation: - 5.5"}],"model":"gpt-4o-2024-08-06","n":1,"stop":["Observation:"]}' + The 7 day forecast for New York is [...]"}, {"role": "user", "content": "Cast + ''5.5'' to a float, then to an integer, and finally use it to write a story + of that many words."}, {"role": "assistant", "content": "Thought: I need to + first cast ''5.5'' to a float.\nAction: cast_float\nAction Input: (''5.5'',)"}, + {"role": "user", "content": "Observation: 5.5"}], "model": "gpt-4o-mini-2024-07-18", + "n": 1, "stop": ["Observation:"]}' headers: accept: - application/json @@ -145,13 +147,13 @@ interactions: connection: - keep-alive content-length: - - "1514" + - "1528" content-type: - application/json host: - api.openai.com user-agent: - - AsyncOpenAI/Python 1.56.2 + - AsyncOpenAI/Python 1.59.3 x-stainless-arch: - arm64 x-stainless-async: @@ -161,7 +163,7 @@ interactions: x-stainless-os: - MacOS x-stainless-package-version: - - 1.56.2 + - 1.59.3 x-stainless-raw-response: - "true" x-stainless-retry-count: @@ -175,19 +177,20 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAAAwAAAP//jJJNa9wwEIbv/hWDTi3sLu5+r2+BUthLD20oLU0xWmlsq5U1ijRuG8L+ - 9yLvhx2aQC7GvM/My7wzeswAhNGiAKEayar1dnqj809V83B//0HHfW0Pn7/c0pY2X/ff3ru1mKQO - OvxExZeumaLWW2RD7oRVQMmYXN9tFovtarddb3rQkkab2mrP0yVN5/l8Oc230/zsqxoyCqMo4HsG - APDYf9OITuNfUUA+uSgtxihrFMW1CEAEskkRMkYTWToWkwEqcoyun/q2oa5uuICP9Ae4kQx7aORv - BG4QKkuSYTVbTWAPDlEDEygZGQynX+nAOMYaw+zO3agUu+h5aRxfFNg733EBb5LP2/EYAasuyrQF - 11l71o/XXJZqH+gQz/yqV8aZ2JQBZSSXMkQmL3p6zAB+9PvrnqxE+ECt55LpF7pkuJjvTn5iuNiI - rs6QiaUd6evl5Bm/UiNLY+PoAkJJ1aAeWodzyU4bGoFslPr/aZ7zPiU3rn6N/QCUQs+oSx9QG/U0 - 8VAWMD3ol8quW+4HFvEhMrZlZVyNwQdzelOVL3d6lSu92+UHkR2zfwAAAP//AwBU5kw4XAMAAA== + H4sIAAAAAAAAAwAAAP//jFJNi9swFLz7Vzx0aiEOsZ2v9S2lh00pZRdKobTFKPKzrVaWVOk527Lk + vxfZSeylu7AXHWbeDG/m6TECYLJkOTDRcBKtVfHO1kf9Tu2Wv1e3H77cdYf7e/7xfZKubr/e7dgs + KMzhJwq6qObCtFYhSaMHWjjkhME12WTr7TZLkpueaE2JKshqS/HSxK3UMk4X6TJebOJke1Y3Rgr0 + LIdvEQDAY/+GPXWJf1gOi9kFadF7XiPLr0MAzBkVEMa9l564JjYbSWE0oe5X/9yYrm4oh0/mAajh + BHto+BGBGoRKGU6wmq9msIcHqRQI7gkkARngGqQmrNHNv+udCMHzni+kpgsCe207yuFNMHk73cFh + 1XkeetCdUmf8dA2lTG2dOfgzf8UrqaVvCofcGx0CeDKW9ewpAvjRl9c96YNZZ1pLBZlfqINhlm4G + PzbebMIuzyQZ4mqCr5PZM35FicSl8pP6meCiwXKUjrfiXSnNhIgmqf/f5jnvIbnU9WvsR0IItIRl + YR2WUjxNPI45DF/6pbFry/3CzKM7SoEFSXThEiVWvFPDR2P+rydsi0rqGp11cvhtlS0O5TbL0ptq + nbHoFP0DAAD//wMAkQFlTHsDAAA= headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8effbacc28d6d039-SJC + - 90200ca6fd7267f0-SJC Connection: - keep-alive Content-Encoding: @@ -195,7 +198,7 @@ interactions: Content-Type: - application/json Date: - - Tue, 10 Dec 2024 19:44:28 GMT + - Tue, 14 Jan 2025 19:32:00 GMT Server: - cloudflare Transfer-Encoding: @@ -209,31 +212,31 @@ interactions: openai-organization: - future-house-xr4tdh openai-processing-ms: - - "625" + - "1132" openai-version: - "2020-10-01" strict-transport-security: - max-age=31536000; includeSubDomains; preload x-ratelimit-limit-requests: - - "10000" + - "30000" x-ratelimit-limit-tokens: - - "30000000" + - "150000000" x-ratelimit-remaining-requests: - - "9999" + - "29999" x-ratelimit-remaining-tokens: - - "29999662" + - "149999666" x-ratelimit-reset-requests: - - 6ms + - 2ms x-ratelimit-reset-tokens: - 0s x-request-id: - - req_21906eb43abccec10d343df6ef16ef82 + - req_517016d8b77667a507775cfcbec04424 status: code: 200 message: OK - request: body: - '{"messages":[{"role":"system","content":"Answer the following questions + '{"messages": [{"role": "system", "content": "Answer the following questions as best you can. You have access to the following tools:\n\nNAME: print_story\n\nSYNOPSIS:\n print_story(string story)\n\nDESCRIPTION:\n Print a story.\n\nPARAMETERS:\n story (string): Story to print.\n\nNAME: cast_float\n\nSYNOPSIS:\n cast_float(string x)\n\nDESCRIPTION:\n Cast @@ -246,13 +249,14 @@ interactions: tuple\nObservation: the result of the action\n... (this Thought/Action/Action Input/Observation can repeat N times)\n\nExample:\n\nThought: I need to use the get_weather tool\nAction: get_weather\nAction Input: \"New York\", 7\nObservation: - The 7 day forecast for New York is [...]"},{"role":"user","content":"Cast ''5.5'' - to a float, then to an integer, and finally use it to write a story of that - many words."},{"role":"assistant","content":"Thought: I need to first cast the - string ''5.5'' to a float.\nAction: cast_float\nAction Input: (\"5.5\",)"},{"role":"user","content":"Observation: - 5.5"},{"role":"assistant","content":"Thought: Now that I have the float 5.5, - I need to cast it to an integer.\nAction: cast_int\nAction Input: (5.5,)"},{"role":"user","content":"Observation: - 5"}],"model":"gpt-4o-2024-08-06","n":1,"stop":["Observation:"]}' + The 7 day forecast for New York is [...]"}, {"role": "user", "content": "Cast + ''5.5'' to a float, then to an integer, and finally use it to write a story + of that many words."}, {"role": "assistant", "content": "Thought: I need to + first cast ''5.5'' to a float.\nAction: cast_float\nAction Input: (''5.5'',)"}, + {"role": "user", "content": "Observation: 5.5"}, {"role": "assistant", "content": + "Thought: Now that I have the float 5.5, I will cast it to an integer.\nAction: + cast_int\nAction Input: (5.5,)"}, {"role": "user", "content": "Observation: + 5"}], "model": "gpt-4o-mini-2024-07-18", "n": 1, "stop": ["Observation:"]}' headers: accept: - application/json @@ -261,13 +265,13 @@ interactions: connection: - keep-alive content-length: - - "1703" + - "1722" content-type: - application/json host: - api.openai.com user-agent: - - AsyncOpenAI/Python 1.56.2 + - AsyncOpenAI/Python 1.59.3 x-stainless-arch: - arm64 x-stainless-async: @@ -277,7 +281,7 @@ interactions: x-stainless-os: - MacOS x-stainless-package-version: - - 1.56.2 + - 1.59.3 x-stainless-raw-response: - "true" x-stainless-retry-count: @@ -291,20 +295,21 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAAAwAAAP//jFNNj9owEL3nV4x8aqWAsgsskBu3UlVtVe2hUllFxh4SU8fj2hNYuuK/ - VwkfYdWt1EsO7817evPGeUkAhNEiB6Eqyar2drDQ2bdyPJ/W9PHX9PsHre1uQb/11+1i++lRpK2C - 1ltUfFENFdXeIhtyJ1oFlIyt6910NJpN5rOHWUfUpNG2stLzYEyD++x+PMhmg+zhLKzIKIwihx8J - AMBL920jOo3PIocsvSA1xihLFPl1CEAEsi0iZIwmsnQs0p5U5Bhdl/qxoqasOIfPtAeuJMMSKrlD - 4ArBOMYSA+ykbRAmKSxhb6yFfTCMICEyhcNJZSLgs1RsDzCBPQUdwZIrhyu3UG0dOfhgHBed5ALC - 0vmGc3i3El+cQmg8OZDApsYUKum9cRjjcCXe36YPuGmibMtzjbVn/Hitw1LpA63jmb/iG+NMrIqA - MpJrV49MXnTsMQF46mpvXjUpfKDac8H0E11rOJo+nPxEf+ieHd+dSSaW9hafpm/4FRpZGhtvDieU - VBXqXtpfWTba0A2R3Gz9d5q3vE+bG1f+j31PKIWeURc+oDbq9cb9WMD2P/jX2LXlLrCIh8hYFxvj - Sgzdu2jvsfHFXE8ypefzbC2SY/IHAAD//wMAOh6TDpMDAAA= + H4sIAAAAAAAAAwAAAP//jFPBjtowEL3zFSNftpUAQQIbyG2lXlCr7WUPlbpVZOxJ4l3H49oTKFrx + 71UCS0BtpV5yeO/N08x7ztsIQBgtchCqlqwabycPvtpReMRo5cvnUP58Sg5Z9rquPn37wmsx7iZo + +4KK36emihpvkQ25E60CSsbOdZ6l96tVOk9mPdGQRtuNVZ4nC5o0xplJMksWk1k2ma/O0zUZhVHk + 8H0EAPDWf7s9ncZfIofeq0cajFFWKPKLCEAEsh0iZIwmsnQsxgOpyDG6fvWnmtqq5hw2UMsdQmyV + whjL1toDKHI7DIwa7pbT5R0wAdcIxjFWGGA5hUfaj2EDe2Mt7INhBAmRKRyASljCnoKO02f3oLpc + cvDBOC56wTsIG+dbzuHDs/jqFELryYEENg2OoZGVUVBL79Ghnj6L8cfrOwKWbZRdlq619owfL8FY + qnygbTzzF7w0zsS6CCgjuS6EyORFzx5HAD/6AtqbTIUP1HgumF7RdYZplp78xND7wC6yM8nE0l7h + ybm2W79CI0tj41WFQklVox5Gh75lqw1dEaOrq//c5m/ep8uNq/7HfiCUQs+oCx9QG3V78SAL2P0W + /5JdUu4XFhHDzigs2GDomtBYytaeHquIh8jYFKVxFYb+6XSS0hdbvUrTZF3ep2J0HP0GAAD//wMA + Kksaur8DAAA= headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8effbad0cd60d039-SJC + - 90200caf0bb767f0-SJC Connection: - keep-alive Content-Encoding: @@ -312,7 +317,7 @@ interactions: Content-Type: - application/json Date: - - Tue, 10 Dec 2024 19:44:29 GMT + - Tue, 14 Jan 2025 19:32:01 GMT Server: - cloudflare Transfer-Encoding: @@ -326,25 +331,25 @@ interactions: openai-organization: - future-house-xr4tdh openai-processing-ms: - - "888" + - "1216" openai-version: - "2020-10-01" strict-transport-security: - max-age=31536000; includeSubDomains; preload x-ratelimit-limit-requests: - - "10000" + - "30000" x-ratelimit-limit-tokens: - - "30000000" + - "150000000" x-ratelimit-remaining-requests: - - "9999" + - "29999" x-ratelimit-remaining-tokens: - - "29999630" + - "149999633" x-ratelimit-reset-requests: - - 6ms + - 2ms x-ratelimit-reset-tokens: - 0s x-request-id: - - req_d1f9a64806371bef60e98109cc7df15e + - req_337ec9e0489a1923c98c0a5033101dca status: code: 200 message: OK diff --git a/tests/test_agents.py b/tests/test_agents.py index db637b82..399f0957 100644 --- a/tests/test_agents.py +++ b/tests/test_agents.py @@ -11,13 +11,13 @@ from aviary.core import DummyEnv, Message, Tool, ToolCall, ToolRequestMessage from aviary.message import EnvStateMessage from httpx import ASGITransport, AsyncClient +from llmclient import CommonLLMNames from llmclient import MultipleCompletionLLMModel as LLMModel from pydantic import BaseModel, Field from ldp.agent import ( Agent, AgentConfig, - DefaultLLMModelNames, HTTPAgentClient, MemoryAgent, ReActAgent, @@ -36,7 +36,6 @@ ToolDescriptionMethods, ) -from . import CILLMModelNames from .conftest import IN_GITHUB_ACTIONS, VCR_DEFAULT_MATCH_ON HERE = Path(__file__).parent @@ -118,7 +117,8 @@ def test_serialization_deserializaton(self) -> None: class TestSimpleAgent: @pytest.mark.parametrize( - "model_name", [CILLMModelNames.ANTHROPIC.value, CILLMModelNames.OPENAI.value] + "model_name", + [CommonLLMNames.ANTHROPIC_TEST.value, CommonLLMNames.OPENAI_TEST.value], ) @pytest.mark.asyncio @pytest.mark.vcr @@ -153,7 +153,8 @@ async def test_dummyenv(self, dummy_env: DummyEnv, model_name: str) -> None: raise RuntimeError("Could not find LLMCallOp in compute graph") @pytest.mark.parametrize( - "model_name", [CILLMModelNames.ANTHROPIC.value, CILLMModelNames.OPENAI.value] + "model_name", + [CommonLLMNames.ANTHROPIC_TEST.value, CommonLLMNames.OPENAI_TEST.value], ) @pytest.mark.asyncio @pytest.mark.vcr @@ -232,7 +233,7 @@ async def test_hide_old_env_states(self) -> None: class TestMemoryAgent: # # On 5/14/2024, claude 3 opus would not follow its past memories - @pytest.mark.parametrize("model_name", [CILLMModelNames.OPENAI.value]) + @pytest.mark.parametrize("model_name", [CommonLLMNames.OPENAI_TEST.value]) @pytest.mark.asyncio @pytest.mark.vcr async def test_dummyenv(self, dummy_env: DummyEnv, model_name: str) -> None: @@ -325,8 +326,8 @@ class TestReActAgent: @pytest.mark.parametrize( ("single_prompt", "model_name"), [ - (True, CILLMModelNames.ANTHROPIC.value), - (False, CILLMModelNames.ANTHROPIC.value), + (True, CommonLLMNames.ANTHROPIC_TEST.value), + (False, CommonLLMNames.ANTHROPIC_TEST.value), (True, "gpt-4-turbo"), (False, "gpt-4o"), ], @@ -376,7 +377,7 @@ async def test_multi_step(self, dummy_env: DummyEnv, single_prompt: bool) -> Non agent = ReActAgent( single_prompt=single_prompt, llm_model={ - "model": DefaultLLMModelNames.OPENAI.value, + "model": CommonLLMNames.OPENAI_TEST.value, # If tools are provided, don't allow it to make parallel tool calls, since # we want to force longer trajectories. In single_prompt mode, parallel tool # calling is not possible, and OpenAI requires parallel_tool_calls=None @@ -421,8 +422,8 @@ def test_agent_op_naming(self, single_prompt: bool) -> None: @pytest.mark.parametrize( ("single_prompt", "model_name"), [ - (True, CILLMModelNames.ANTHROPIC.value), - (False, CILLMModelNames.ANTHROPIC.value), + (True, CommonLLMNames.ANTHROPIC_TEST.value), + (False, CommonLLMNames.ANTHROPIC_TEST.value), (True, "gpt-4-turbo"), (False, "gpt-4o"), ], diff --git a/tests/test_envs.py b/tests/test_envs.py index e975b96f..d1ecce45 100644 --- a/tests/test_envs.py +++ b/tests/test_envs.py @@ -12,11 +12,10 @@ ToolRequestMessage, ToolResponseMessage, ) +from llmclient import CommonLLMNames from ldp.agent import SimpleAgent -from . import CILLMModelNames - class ParallelizedDummyEnv(DummyEnv): def __init__(self, right_hand_broken: bool = False): @@ -105,7 +104,7 @@ async def test_SimpleAgent_can_parallel_call(self) -> None: ) @pytest.mark.parametrize( - "model_name", [CILLMModelNames.ANTHROPIC.value, "gpt-4-turbo"] + "model_name", [CommonLLMNames.ANTHROPIC_TEST.value, "gpt-4-turbo"] ) @pytest.mark.asyncio async def test_exec_tool_calls_handling(self, model_name: str) -> None: diff --git a/tests/test_modules.py b/tests/test_modules.py index e15d5453..f6e4c250 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -2,6 +2,7 @@ import pytest from aviary.core import DummyEnv, Message, Tool, ToolRequestMessage +from llmclient import CommonLLMNames from pytest_subtests import SubTests from ldp.agent import ReActAgent @@ -14,15 +15,16 @@ parse_message, ) -from . import CILLMModelNames - @pytest.mark.asyncio @pytest.mark.vcr async def test_reflect_module() -> None: config = ReflectModuleConfig( - llm_model={"model": CILLMModelNames.ANTHROPIC.value, "temperature": 0} - ) # Lower temperature for more deterministic responses + llm_model={ + "model": CommonLLMNames.ANTHROPIC_TEST.value, + "temperature": 0, # Lower temperature for more deterministic responses + } + ) reflect_module = ReflectModule(config) context = "I am happy. How do I feel?" response = "You are sad." diff --git a/tests/test_ops.py b/tests/test_ops.py index b1585d4a..888e43bd 100644 --- a/tests/test_ops.py +++ b/tests/test_ops.py @@ -9,6 +9,7 @@ import pytest import tree from aviary.core import DummyEnv, Message, Tool, ToolRequestMessage +from llmclient import CommonLLMNames from llmclient import MultipleCompletionLLMModel as LLMModel from ldp.graph import ( @@ -173,8 +174,7 @@ async def test_empty_tools(self) -> None: @pytest.mark.asyncio @pytest.mark.parametrize("temperature", [0.0, 0.5, 1.0]) async def test_compute_logprob(self, temperature) -> None: - model_name = "gpt-4o-mini" - config = {"model": model_name, "temperature": temperature} + config = {"model": CommonLLMNames.OPENAI_TEST.value, "temperature": temperature} llm_op = LLMCallOp() output = await llm_op(config, msgs=[Message(content="Hello")]) logp = llm_op.ctx.get(output.call_id, "logprob") diff --git a/tests/test_optimizer.py b/tests/test_optimizer.py index 67e1eb22..d6cd71cd 100644 --- a/tests/test_optimizer.py +++ b/tests/test_optimizer.py @@ -7,6 +7,7 @@ import tenacity import tree from aviary.core import Message +from llmclient import CommonLLMNames from llmclient import MultipleCompletionLLMModel as LLMModel from pydantic import BaseModel, Field, JsonValue @@ -40,7 +41,6 @@ ) from ldp.graph.ops import GradInType from ldp.llms.prompts import append_to_sys -from tests import CILLMModelNames from tests.conftest import VCR_DEFAULT_MATCH_ON @@ -297,7 +297,7 @@ async def test_lessons_memory_optimizer(self) -> None: This test is loosely based on Reflexion (https://arxiv.org/abs/2303.11366). """ - memory_distiller = LLMModel(config={"model": CILLMModelNames.OPENAI.value}) + memory_distiller = LLMModel(config={"model": CommonLLMNames.OPENAI_TEST.value}) class LessonEntry(BaseModel): """Entry for a lesson created from some example data.""" From 1b8d05f7851d1f8d7af5c5d83bed00f39bfe83d7 Mon Sep 17 00:00:00 2001 From: James Braza Date: Tue, 14 Jan 2025 11:40:41 -0800 Subject: [PATCH 2/2] Moved agents to use CommonLLMNames.GPT_4O per request --- ldp/agent/react_agent.py | 2 +- ldp/agent/simple_agent.py | 2 +- ldp/agent/tree_of_thoughts_agent.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ldp/agent/react_agent.py b/ldp/agent/react_agent.py index 027799c6..d160e874 100644 --- a/ldp/agent/react_agent.py +++ b/ldp/agent/react_agent.py @@ -82,7 +82,7 @@ class ReActAgent(BaseModel, Agent[SimpleAgentState]): llm_model: dict[str, Any] = Field( default={ - "model": CommonLLMNames.OPENAI_BASELINE.value, + "model": CommonLLMNames.GPT_4O.value, "temperature": 0.1, "logprobs": True, "top_logprobs": 1, diff --git a/ldp/agent/simple_agent.py b/ldp/agent/simple_agent.py index ce7b329a..c3eaa96e 100644 --- a/ldp/agent/simple_agent.py +++ b/ldp/agent/simple_agent.py @@ -84,7 +84,7 @@ class SimpleAgent(BaseModel, Agent[SimpleAgentState]): llm_model: dict[str, Any] = Field( default={ - "model": CommonLLMNames.OPENAI_BASELINE.value, + "model": CommonLLMNames.GPT_4O.value, "temperature": 0.1, "timeout": DEFAULT_LLM_COMPLETION_TIMEOUT, }, diff --git a/ldp/agent/tree_of_thoughts_agent.py b/ldp/agent/tree_of_thoughts_agent.py index 1e232963..76ac7683 100644 --- a/ldp/agent/tree_of_thoughts_agent.py +++ b/ldp/agent/tree_of_thoughts_agent.py @@ -44,7 +44,7 @@ class TreeofThoughtsAgent(BaseModel, Agent[SimpleAgentState]): llm_model: dict[str, Any] = Field( default={ - "model": CommonLLMNames.OPENAI_BASELINE.value, + "model": CommonLLMNames.GPT_4O.value, "temperature": 0.1, "timeout": DEFAULT_LLM_COMPLETION_TIMEOUT, },