From bc6b37f74e58a7c1d9164b02be53f0a71d8ab45f Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Mon, 25 Mar 2024 10:32:30 +0100 Subject: [PATCH 1/3] More prints to debug known outcome agent --- .../agents/known_outcome_agent/deploy.py | 9 +++++++++ .../agents/known_outcome_agent/known_outcome_agent.py | 2 ++ 2 files changed, 11 insertions(+) diff --git a/prediction_market_agent/agents/known_outcome_agent/deploy.py b/prediction_market_agent/agents/known_outcome_agent/deploy.py index d364ed31..90380c05 100644 --- a/prediction_market_agent/agents/known_outcome_agent/deploy.py +++ b/prediction_market_agent/agents/known_outcome_agent/deploy.py @@ -37,15 +37,24 @@ def pick_markets(self, markets: list[AgentMarket]) -> list[AgentMarket]: # been correctly bet on, and therefore the value of betting on them # is low. if not market_is_saturated(market=market): + print(f"Checking market {market.id=} {market.question=}") answer = get_known_outcome( model=self.model, question=market.question, max_tries=3, ) if answer.has_known_outcome(): + print( + f"Picking market {market.id=} {market.question=} with answer {answer.result=}" + ) picked_markets.append(market) self.markets_with_known_outcomes[market.id] = answer.result + else: + print( + f"Skipping market {market.id=} {market.question=}, because it is already saturated." + ) + return picked_markets def answer_binary_market(self, market: AgentMarket) -> bool: diff --git a/prediction_market_agent/agents/known_outcome_agent/known_outcome_agent.py b/prediction_market_agent/agents/known_outcome_agent/known_outcome_agent.py index 983d5bf9..1a5d5299 100644 --- a/prediction_market_agent/agents/known_outcome_agent/known_outcome_agent.py +++ b/prediction_market_agent/agents/known_outcome_agent/known_outcome_agent.py @@ -152,7 +152,9 @@ def get_known_outcome(model: str, question: str, max_tries: int) -> Answer: search_prompt = ChatPromptTemplate.from_template( template=GENERATE_SEARCH_QUERY_PROMPT ).format_messages(date_str=date_str, question=question) + print(f"Invoking LLM for {search_prompt=}") search_query = str(llm.invoke(search_prompt).content).strip('"') + print(f"Searchig for {search_query=}") search_results = web_search(query=search_query, max_results=5) if not search_results: raise ValueError("No search results found.") From 895086834c84adc852cb7d3e20980780e4bcc8ab Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Mon, 25 Mar 2024 10:36:23 +0100 Subject: [PATCH 2/3] fix prints --- prediction_market_agent/agents/known_outcome_agent/deploy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/prediction_market_agent/agents/known_outcome_agent/deploy.py b/prediction_market_agent/agents/known_outcome_agent/deploy.py index 90380c05..a01e4721 100644 --- a/prediction_market_agent/agents/known_outcome_agent/deploy.py +++ b/prediction_market_agent/agents/known_outcome_agent/deploy.py @@ -36,8 +36,9 @@ def pick_markets(self, markets: list[AgentMarket]) -> list[AgentMarket]: # Assume very high probability markets are already known, and have # been correctly bet on, and therefore the value of betting on them # is low. + print(f"Looking at market {market.id=} {market.question=}") if not market_is_saturated(market=market): - print(f"Checking market {market.id=} {market.question=}") + print(f"Predicting market {market.id=} {market.question=}") answer = get_known_outcome( model=self.model, question=market.question, From 57a1a0b135664ec303093c50a513e6a966e4071e Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Tue, 2 Apr 2024 08:37:59 +0200 Subject: [PATCH 3/3] Skip failed predictions in known outcome agent and bet only on 1 market (#38) --- .../agents/known_outcome_agent/deploy.py | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/prediction_market_agent/agents/known_outcome_agent/deploy.py b/prediction_market_agent/agents/known_outcome_agent/deploy.py index a01e4721..7a14e590 100644 --- a/prediction_market_agent/agents/known_outcome_agent/deploy.py +++ b/prediction_market_agent/agents/known_outcome_agent/deploy.py @@ -39,11 +39,17 @@ def pick_markets(self, markets: list[AgentMarket]) -> list[AgentMarket]: print(f"Looking at market {market.id=} {market.question=}") if not market_is_saturated(market=market): print(f"Predicting market {market.id=} {market.question=}") - answer = get_known_outcome( - model=self.model, - question=market.question, - max_tries=3, - ) + try: + answer = get_known_outcome( + model=self.model, + question=market.question, + max_tries=3, + ) + except Exception as e: + print( + f"Error: Failed to predict market {market.id=} {market.question=}: {e}" + ) + continue if answer.has_known_outcome(): print( f"Picking market {market.id=} {market.question=} with answer {answer.result=}" @@ -51,6 +57,11 @@ def pick_markets(self, markets: list[AgentMarket]) -> list[AgentMarket]: picked_markets.append(market) self.markets_with_known_outcomes[market.id] = answer.result + # Return as soon as we have picked a market, because otherwise it will take too long and we will run out of time in GCP Function (540s timeout) + # TODO: After PMAT is updated in this repository, we can return `None` in `answer_binary_market` method and PMAT won't place the bet. + # So we can move this logic out of `pick_markets` into `answer_binary_market`, and simply process as many bets as we have time for. + return picked_markets + else: print( f"Skipping market {market.id=} {market.question=}, because it is already saturated."