Merge pull request hummingbot#7214 from cardosofede/feat/improve_mark…

…et_data_provider

Feat/improve market data provider

hummingbot/connector/derivative/okx_perpetual/okx_perpetual_derivative.py

@@ -129,7 +129,7 @@ def supported_order_types(self) -> List[OrderType]:
         :return a list of OrderType supported by this connector
         """
         # TODO: Check if it's market or limit_maker
-        return [OrderType.LIMIT, OrderType.MARKET]
+        return [OrderType.LIMIT, OrderType.MARKET, OrderType.LIMIT_MAKER]
 
     def supported_position_modes(self) -> List[PositionMode]:
         return [PositionMode.ONEWAY, PositionMode.HEDGE]
@@ -316,7 +316,7 @@ async def get_last_traded_prices(self, trading_pairs: List[str] = None) -> Dict[
             params=params,
         )
 
-        last_traded_prices = {ticker["instId"]: float(ticker["last"]) for ticker in resp_json["data"]}
+        last_traded_prices = {ticker["instId"].replace("-SWAP", ""): float(ticker["last"]) for ticker in resp_json["data"]}
         return last_traded_prices
 
     async def _update_balances(self):

hummingbot/connector/exchange/okx/okx_constants.py

@@ -19,6 +19,7 @@
 OKX_SERVER_TIME_PATH = '/api/v5/public/time'
 OKX_INSTRUMENTS_PATH = '/api/v5/public/instruments'
 OKX_TICKER_PATH = '/api/v5/market/ticker'
+OKX_TICKERS_PATH = '/api/v5/market/tickers'
 OKX_ORDER_BOOK_PATH = '/api/v5/market/books'
 
 # Auth required
@@ -71,6 +72,7 @@
     RateLimit(limit_id=OKX_SERVER_TIME_PATH, limit=10, time_interval=2),
     RateLimit(limit_id=OKX_INSTRUMENTS_PATH, limit=20, time_interval=2),
     RateLimit(limit_id=OKX_TICKER_PATH, limit=20, time_interval=2),
+    RateLimit(limit_id=OKX_TICKERS_PATH, limit=20, time_interval=2),
     RateLimit(limit_id=OKX_ORDER_BOOK_PATH, limit=20, time_interval=2),
     RateLimit(limit_id=OKX_PLACE_ORDER_PATH, limit=20, time_interval=2),
     RateLimit(limit_id=OKX_ORDER_DETAILS_PATH, limit=20, time_interval=2),

hummingbot/connector/exchange/okx/okx_exchange.py

@@ -237,6 +237,17 @@ async def _place_cancel(self, order_id: str, tracked_order: InFlightOrder):
 
         return final_result
 
+    async def get_last_traded_prices(self, trading_pairs: List[str] = None) -> Dict[str, float]:
+        params = {"instType": "SPOT"}
+
+        resp_json = await self._api_get(
+            path_url=CONSTANTS.OKX_TICKERS_PATH,
+            params=params,
+        )
+
+        last_traded_prices = {ticker["instId"]: float(ticker["last"]) for ticker in resp_json["data"]}
+        return last_traded_prices
+
     async def _get_last_traded_price(self, trading_pair: str) -> float:
         params = {"instId": await self.exchange_symbol_associated_to_pair(trading_pair=trading_pair)}
 

hummingbot/data_feed/market_data_provider.py

@@ -1,4 +1,5 @@
 import time
+from decimal import Decimal
 from typing import Dict, List, Tuple
 
 import pandas as pd
@@ -144,6 +145,23 @@ def get_trading_pairs(self, connector_name: str):
         connector = self.get_connector(connector_name)
         return connector.trading_pairs
 
+    def get_trading_rules(self, connector_name: str, trading_pair: str):
+        """
+        Retrieves the trading rules from the specified connector.
+        :param connector_name: str
+        :return: Trading rules.
+        """
+        connector = self.get_connector(connector_name)
+        return connector.trading_rules[trading_pair]
+
+    def quantize_order_price(self, connector_name: str, trading_pair: str, price: Decimal):
+        connector = self.get_connector(connector_name)
+        return connector.quantize_order_price(trading_pair, price)
+
+    def quantize_order_amount(self, connector_name: str, trading_pair: str, amount: Decimal):
+        connector = self.get_connector(connector_name)
+        return connector.quantize_order_amount(trading_pair, amount)
+
     def get_price_for_volume(self, connector_name: str, trading_pair: str, volume: float,
                              is_buy: bool) -> OrderBookQueryResult:
         """

hummingbot/strategy_v2/backtesting/backtesting_data_provider.py

@@ -1,33 +1,84 @@
+import logging
 from decimal import Decimal
 from typing import Dict
 
 import pandas as pd
 
+from hummingbot.client.config.client_config_map import ClientConfigMap
+from hummingbot.client.config.config_helpers import ClientConfigAdapter, get_connector_class
+from hummingbot.client.settings import AllConnectorSettings, ConnectorType
 from hummingbot.connector.connector_base import ConnectorBase
 from hummingbot.core.data_type.common import PriceType
 from hummingbot.data_feed.candles_feed.candles_factory import CandlesFactory
 from hummingbot.data_feed.candles_feed.data_types import CandlesConfig, HistoricalCandlesConfig
 from hummingbot.data_feed.market_data_provider import MarketDataProvider
 
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
 
 class BacktestingDataProvider(MarketDataProvider):
+    CONNECTOR_TYPES = [ConnectorType.CLOB_SPOT, ConnectorType.CLOB_PERP, ConnectorType.Exchange,
+                       ConnectorType.Derivative]
+    EXCLUDED_CONNECTORS = ["vega_perpetual", "hyperliquid_perpetual", "dydx_perpetual", "cube",
+                           "polkadex", "coinbase_advanced_trade", "kraken", "dydx_v4_perpetual", "hitbtc"]
+
     def __init__(self, connectors: Dict[str, ConnectorBase]):
         super().__init__(connectors)
         self.start_time = None
         self.end_time = None
         self.prices = {}
         self._time = None
+        self.trading_rules = {}
+        self.conn_settings = AllConnectorSettings.get_connector_settings()
+        self.connectors = {name: self.get_connector(name) for name, settings in self.conn_settings.items()
+                           if settings.type in self.CONNECTOR_TYPES and name not in self.EXCLUDED_CONNECTORS and
+                           "testnet" not in name}
+
+    def get_connector(self, connector_name: str):
+        conn_setting = self.conn_settings.get(connector_name)
+        if conn_setting is None:
+            logger.error(f"Connector {connector_name} not found")
+            raise ValueError(f"Connector {connector_name} not found")
+
+        client_config_map = ClientConfigAdapter(ClientConfigMap())
+        init_params = conn_setting.conn_init_parameters(
+            trading_pairs=[],
+            trading_required=False,
+            api_keys=self.get_connector_config_map(connector_name),
+            client_config_map=client_config_map,
+        )
+        connector_class = get_connector_class(connector_name)
+        connector = connector_class(**init_params)
+        return connector
+
+    @staticmethod
+    def get_connector_config_map(connector_name: str):
+        connector_config = AllConnectorSettings.get_connector_config_keys(connector_name)
+        return {key: "" for key in connector_config.__fields__.keys() if key != "connector"}
+
+    def get_trading_rules(self, connector_name: str, trading_pair: str):
+        """
+        Retrieves the trading rules from the specified connector.
+        :param connector_name: str
+        :return: Trading rules.
+        """
+        return self.trading_rules[connector_name][trading_pair]
 
     def time(self):
         return self._time
 
+    async def initialize_trading_rules(self, connector_name: str):
+        if len(self.trading_rules.get(connector_name, {})) == 0:
+            connector = self.connectors.get(connector_name)
+            await connector._update_trading_rules()
+            self.trading_rules[connector_name] = connector.trading_rules
+
     async def initialize_candles_feed(self, config: CandlesConfig):
         await self.get_candles_feed(config)
 
     def update_backtesting_time(self, start_time: int, end_time: int):
-        if (self.start_time is None or self.end_time is None) or \
-                (start_time < self.start_time or end_time > self.end_time):
-            self.candles_feeds = {}
         self.start_time = start_time
         self.end_time = end_time
         self._time = start_time
@@ -43,20 +94,21 @@ async def get_candles_feed(self, config: CandlesConfig):
         existing_feed = self.candles_feeds.get(key, pd.DataFrame())
 
         if not existing_feed.empty:
-            # Existing feed is sufficient, return it
-            return existing_feed
-        else:
-            # Create a new feed or restart the existing one with updated max_records
-            candle_feed = CandlesFactory.get_candle(config)
-            candles_df = await candle_feed.get_historical_candles(config=HistoricalCandlesConfig(
-                connector_name=config.connector,
-                trading_pair=config.trading_pair,
-                interval=config.interval,
-                start_time=self.start_time,
-                end_time=self.end_time,
-            ))
-            self.candles_feeds[key] = candles_df
-            return candles_df
+            existing_feed_start_time = existing_feed["timestamp"].min()
+            existing_feed_end_time = existing_feed["timestamp"].max()
+            if existing_feed_start_time <= self.start_time and existing_feed_end_time >= self.end_time:
+                return existing_feed
+        # Create a new feed or restart the existing one with updated max_records
+        candle_feed = CandlesFactory.get_candle(config)
+        candles_df = await candle_feed.get_historical_candles(config=HistoricalCandlesConfig(
+            connector_name=config.connector,
+            trading_pair=config.trading_pair,
+            interval=config.interval,
+            start_time=self.start_time,
+            end_time=self.end_time,
+        ))
+        self.candles_feeds[key] = candles_df
+        return candles_df
 
     def get_candles_df(self, connector_name: str, trading_pair: str, interval: str, max_records: int = 500):
         """
@@ -79,3 +131,27 @@ def get_price_by_type(self, connector_name: str, trading_pair: str, price_type:
         :return: Price.
         """
         return self.prices.get(f"{connector_name}_{trading_pair}", Decimal("1"))
+
+    def quantize_order_amount(self, connector_name: str, trading_pair: str, amount: Decimal):
+        """
+        Quantizes the order amount based on the trading pair's minimum order size.
+        :param connector_name: str
+        :param trading_pair: str
+        :param amount: Decimal
+        :return: Quantized amount.
+        """
+        trading_rules = self.get_trading_rules(connector_name, trading_pair)
+        order_size_quantum = trading_rules.min_base_amount_increment
+        return (amount // order_size_quantum) * order_size_quantum
+
+    def quantize_order_price(self, connector_name: str, trading_pair: str, price: Decimal):
+        """
+        Quantizes the order price based on the trading pair's minimum price increment.
+        :param connector_name: str
+        :param trading_pair: str
+        :param price: Decimal
+        :return: Quantized price.
+        """
+        trading_rules = self.get_trading_rules(connector_name, trading_pair)
+        price_quantum = trading_rules.min_price_increment
+        return (price // price_quantum) * price_quantum

hummingbot/strategy_v2/backtesting/backtesting_engine_base.py

@@ -85,12 +85,13 @@ async def run_backtesting(self,
         # Load historical candles
         controller_class = controller_config.get_controller_class()
         self.backtesting_data_provider.update_backtesting_time(start, end)
+        await self.backtesting_data_provider.initialize_trading_rules(controller_config.connector_name)
         self.controller = controller_class(config=controller_config, market_data_provider=self.backtesting_data_provider,
                                            actions_queue=None)
         self.backtesting_resolution = backtesting_resolution
         await self.initialize_backtesting_data_provider()
         await self.controller.update_processed_data()
-        executors_info = self.simulate_execution(trade_cost=trade_cost)
+        executors_info = await self.simulate_execution(trade_cost=trade_cost)
         results = self.summarize_results(executors_info, controller_config.total_amount_quote)
         return {
             "executors": executors_info,
@@ -108,7 +109,7 @@ async def initialize_backtesting_data_provider(self):
         for config in self.controller.config.candles_config:
             await self.controller.market_data_provider.initialize_candles_feed(config)
 
-    def simulate_execution(self, trade_cost: float) -> list:
+    async def simulate_execution(self, trade_cost: float) -> list:
         """
         Simulates market making strategy over historical data, considering trading costs.
 
@@ -123,7 +124,7 @@ def simulate_execution(self, trade_cost: float) -> list:
         self.stopped_executors_info: List[ExecutorInfo] = []
         for i, row in processed_features.iterrows():
             self.update_market_data(row)
-            self.update_processed_data(row)
+            await self.update_processed_data(row)
             self.update_executors_info(row["timestamp"])
             for action in self.controller.determine_executor_actions():
                 if isinstance(action, CreateExecutorAction):
@@ -148,7 +149,7 @@ def update_executors_info(self, timestamp: float):
         self.active_executor_simulations = [es for es in self.active_executor_simulations if es.config.id not in simulations_to_remove]
         self.controller.executors_info = active_executors_info + self.stopped_executors_info
 
-    def update_processed_data(self, row: pd.Series):
+    async def update_processed_data(self, row: pd.Series):
         """
         Updates processed data in the controller with the current price and timestamp.
 
@@ -250,14 +251,14 @@ def handle_stop_action(self, action: StopExecutorAction, timestamp: pd.Timestamp
                 self.active_executor_simulations.remove(executor)
 
     @staticmethod
-    def summarize_results(executors_info: Dict, total_amount_quote: float = 1000):
+    def summarize_results(executors_info: List, total_amount_quote: float = 1000):
         if len(executors_info) > 0:
             executors_df = pd.DataFrame([ei.to_dict() for ei in executors_info])
             net_pnl_quote = executors_df["net_pnl_quote"].sum()
             total_executors = executors_df.shape[0]
             executors_with_position = executors_df[executors_df["net_pnl_quote"] != 0]
             total_executors_with_position = executors_with_position.shape[0]
-            total_volume = executors_with_position["filled_amount_quote"].sum() * 2
+            total_volume = executors_with_position["filled_amount_quote"].sum()
             total_long = (executors_with_position["side"] == TradeType.BUY).sum()
             total_short = (executors_with_position["side"] == TradeType.SELL).sum()
             correct_long = ((executors_with_position["side"] == TradeType.BUY) & (executors_with_position["net_pnl_quote"] > 0)).sum()

hummingbot/strategy_v2/backtesting/controllers_backtesting/directional_trading_backtesting.py

@@ -4,5 +4,5 @@
 
 
 class DirectionalTradingBacktesting(BacktestingEngineBase):
-    def update_processed_data(self, row: pd.Series):
+    async def update_processed_data(self, row: pd.Series):
         self.controller.processed_data["signal"] = row["signal"]

hummingbot/strategy_v2/backtesting/controllers_backtesting/market_making_backtesting.py

@@ -6,6 +6,6 @@
 
 
 class MarketMakingBacktesting(BacktestingEngineBase):
-    def update_processed_data(self, row: pd.Series):
+    async def update_processed_data(self, row: pd.Series):
         self.controller.processed_data["reference_price"] = Decimal(row["reference_price"])
         self.controller.processed_data["spread_multiplier"] = Decimal(row["spread_multiplier"])

hummingbot/strategy_v2/backtesting/executors_simulator/dca_executor_simulator.py

@@ -63,11 +63,13 @@ def simulate(self, df: pd.DataFrame, config: DCAExecutorConfig, trade_cost: floa
                 if config.side == TradeType.BUY:
                     ts_activated_condition = returns_df["close"] >= trailing_stop_activation_price
                     if ts_activated_condition.any():
+                        ts_activated_condition = ts_activated_condition.cumsum() > 0
                         returns_df.loc[ts_activated_condition, "ts_trigger_price"] = (returns_df[ts_activated_condition]["close"] * float(1 - trailing_sl_delta_pct)).cummax()
                         trailing_stop_condition = returns_df['close'] <= returns_df['ts_trigger_price']
                 else:
                     ts_activated_condition = returns_df["close"] <= trailing_stop_activation_price
                     if ts_activated_condition.any():
+                        ts_activated_condition = ts_activated_condition.cumsum() > 0
                         returns_df.loc[ts_activated_condition, "ts_trigger_price"] = (returns_df[ts_activated_condition]["close"] * float(1 + trailing_sl_delta_pct)).cummin()
                         trailing_stop_condition = returns_df['close'] >= returns_df['ts_trigger_price']
                 trailing_sl_timestamp = returns_df[trailing_stop_condition]['timestamp'].min() if trailing_stop_condition is not None else None
@@ -135,6 +137,7 @@ def simulate(self, df: pd.DataFrame, config: DCAExecutorConfig, trade_cost: floa
         df_filtered['net_pnl_quote'] = sum([df_filtered[f'net_pnl_quote_{i}'] for i in range(len(potential_dca_stages))])
         df_filtered['cum_fees_quote'] = trade_cost * df_filtered['filled_amount_quote']
         df_filtered.loc[df_filtered["filled_amount_quote"] > 0, "net_pnl_pct"] = df_filtered["net_pnl_quote"] / df_filtered["filled_amount_quote"]
+        df_filtered.loc[df_filtered.index[-1], "filled_amount_quote"] = df_filtered["filled_amount_quote"].iloc[-1] * 2
 
         if close_type is None:
             close_type = CloseType.FAILED