From 8ef7da8a21ef453a236496af787cc2224dc05eba Mon Sep 17 00:00:00 2001
From: EtWn <34377743+EtWnn@users.noreply.github.com>
Date: Mon, 29 Mar 2021 10:14:56 +0200
Subject: [PATCH 1/5] Multiple Accounts (#27)

* add name choice for the Binance account

* add __init__description for BinanceManager
---
 BinanceWatch/BinanceManager.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/BinanceWatch/BinanceManager.py b/BinanceWatch/BinanceManager.py
index 40d7da7..8dd57bb 100644
--- a/BinanceWatch/BinanceManager.py
+++ b/BinanceWatch/BinanceManager.py
@@ -17,8 +17,19 @@ class BinanceManager:
     This class is in charge of filling the database by calling the binance API
     """
 
-    def __init__(self, api_key: str, api_secret: str):
-        self.db = BinanceDataBase()
+    def __init__(self, api_key: str, api_secret: str, account_name: str = 'binance'):
+        """
+        initialise the binance manager.
+
+        :param api_key: key for the Binance api
+        :type api_key: str
+        :param api_secret: secret for the Binance api
+        :type api_secret: str
+        :param account_name: if you have several accounts to monitor, you need to give them different names or the
+        database will collide
+        :type account_name: str
+        """
+        self.db = BinanceDataBase(name=f"{account_name}_db")
         self.client = Client(api_key=api_key, api_secret=api_secret)
 
     def update_spot(self):

From 9912fce12b1c7771d18ae0f50c4a3a5c8cb8cea0 Mon Sep 17 00:00:00 2001
From: EtWn <34377743+EtWnn@users.noreply.github.com>
Date: Mon, 29 Mar 2021 14:01:27 +0200
Subject: [PATCH 2/5] Rate limits (#28)

* Add a logger to the BinanceManager

* add a genereric client call for limit rate

* correct rate limit error number

* integrate rate limit for cross margin api calls

* integrate rate limit for lending api calls

* integrate rate limit for spot api calls
---
 BinanceWatch/BinanceManager.py | 205 +++++++++++++++++++++++++--------
 1 file changed, 158 insertions(+), 47 deletions(-)

diff --git a/BinanceWatch/BinanceManager.py b/BinanceWatch/BinanceManager.py
index 8dd57bb..12ae73d 100644
--- a/BinanceWatch/BinanceManager.py
+++ b/BinanceWatch/BinanceManager.py
@@ -1,13 +1,15 @@
 import datetime
 import math
 import time
-from typing import Optional
+from typing import Optional, Dict
 
 import dateparser
 from binance.client import Client
+from binance.exceptions import BinanceAPIException
 from tqdm import tqdm
 
 from BinanceWatch.storage import tables
+from BinanceWatch.utils.LoggerGenerator import LoggerGenerator
 from BinanceWatch.utils.time_utils import datetime_to_millistamp
 from BinanceWatch.storage.BinanceDataBase import BinanceDataBase
 
@@ -16,8 +18,9 @@ class BinanceManager:
     """
     This class is in charge of filling the database by calling the binance API
     """
+    API_MAX_RETRY = 3
 
-    def __init__(self, api_key: str, api_secret: str, account_name: str = 'binance'):
+    def __init__(self, api_key: str, api_secret: str, account_name: str = 'default'):
         """
         initialise the binance manager.
 
@@ -29,8 +32,10 @@ def __init__(self, api_key: str, api_secret: str, account_name: str = 'binance')
         database will collide
         :type account_name: str
         """
-        self.db = BinanceDataBase(name=f"{account_name}_db")
+        self.account_name = account_name
+        self.db = BinanceDataBase(name=f"{self.account_name}_db")
         self.client = Client(api_key=api_key, api_secret=api_secret)
+        self.logger = LoggerGenerator.get_logger(f"BinanceManager_{self.account_name}")
 
     def update_spot(self):
         """
@@ -98,10 +103,14 @@ def update_universal_transfers(self, transfer_filter: Optional[str] = None):
             latest_time = self.db.get_last_universal_transfer_time(transfer_type=transfer_type) + 1
             current = 1
             while True:
-                universal_transfers = self.client.query_universal_transfer_history(type=transfer_type,
-                                                                                   startTime=latest_time,
-                                                                                   current=current,
-                                                                                   size=100)
+                client_params = {
+                    'type': transfer_type,
+                    'startTime': latest_time,
+                    'current': current,
+                    'size': 100
+                }
+                universal_transfers = self._call_binance_client('query_universal_transfer_history', client_params)
+
                 try:
                     universal_transfers = universal_transfers['rows']
                 except KeyError:
@@ -145,8 +154,15 @@ def update_cross_margin_interests(self):
                 'size': 100,
                 'archived': archived
             }
+
             # no built-in method yet in python-binance for margin/interestHistory
-            interests = self.client._request_margin_api('get', 'margin/interestHistory', signed=True, data=params)
+            client_params = {
+                'method': 'get',
+                'path': 'margin/interestHistory',
+                'signed': True,
+                'data': params
+            }
+            interests = self._call_binance_client('_request_margin_api', client_params)
 
             for interest in interests['rows']:
                 self.db.add_margin_interest(margin_type=margin_type,
@@ -175,7 +191,12 @@ def update_cross_margin_repays(self):
         :return: None
         :rtype: None
         """
-        symbols_info = self.client._request_margin_api('get', 'margin/allPairs', data={})  # not built-in yet
+        client_params = {
+            'method': 'get',
+            'path': 'margin/allPairs',
+            'data': {}
+        }
+        symbols_info = self._call_binance_client('_request_margin_api', client_params)  # not built-in yet
         assets = set()
         for symbol_info in symbols_info:
             assets.add(symbol_info['base'])
@@ -208,12 +229,16 @@ def update_margin_asset_repay(self, asset: str, isolated_symbol=''):
         archived = 1000 * time.time() - latest_time > 1000 * 3600 * 24 * 30 * 3
         current = 1
         while True:
-            repays = self.client.get_margin_repay_details(asset=asset,
-                                                          current=current,
-                                                          startTime=latest_time + 1000,
-                                                          archived=archived,
-                                                          isolatedSymbol=isolated_symbol,
-                                                          size=100)
+            client_params = {
+                'asset': asset,
+                'current':current,
+                'startTime': latest_time + 1000,
+                'archived': archived,
+                'isolatedSymbol': isolated_symbol,
+                'size': 100
+            }
+            repays = self._call_binance_client('get_margin_repay_details', client_params)
+
             for repay in repays['rows']:
                 if repay['status'] == 'CONFIRMED':
                     self.db.add_repay(margin_type=margin_type,
@@ -241,7 +266,12 @@ def update_cross_margin_loans(self):
         :return: None
         :rtype: None
         """
-        symbols_info = self.client._request_margin_api('get', 'margin/allPairs', data={})  # not built-in yet
+        client_params = {
+            'method': 'get',
+            'path': 'margin/allPairs',
+            'data': {}
+        }
+        symbols_info = self._call_binance_client('_request_margin_api', client_params)  # not built-in yet
         assets = set()
         for symbol_info in symbols_info:
             assets.add(symbol_info['base'])
@@ -274,12 +304,16 @@ def update_margin_asset_loans(self, asset: str, isolated_symbol=''):
         archived = 1000 * time.time() - latest_time > 1000 * 3600 * 24 * 30 * 3
         current = 1
         while True:
-            loans = self.client.get_margin_loan_details(asset=asset,
-                                                        current=current,
-                                                        startTime=latest_time + 1000,
-                                                        archived=archived,
-                                                        isolatedSymbol=isolated_symbol,
-                                                        size=100)
+            client_params = {
+                'asset': asset,
+                'current': current,
+                'startTime': latest_time + 1000,
+                'archived': archived,
+                'isolatedSymbol': isolated_symbol,
+                'size': 100
+            }
+            loans = self._call_binance_client('get_margin_loan_details', client_params)
+
             for loan in loans['rows']:
                 if loan['status'] == 'CONFIRMED':
                     self.db.add_loan(margin_type=margin_type,
@@ -321,7 +355,13 @@ def update_cross_margin_symbol_trades(self, asset: str, ref_asset: str, limit: i
         symbol = asset + ref_asset
         last_trade_id = self.db.get_max_trade_id(asset, ref_asset, 'cross_margin')
         while True:
-            new_trades = self.client.get_margin_trades(symbol=symbol, fromId=last_trade_id + 1, limit=limit)
+            client_params = {
+                'symbol': symbol,
+                'fromId': last_trade_id + 1,
+                'limit': limit
+            }
+            new_trades = self._call_binance_client('get_margin_trades', client_params)
+
             for trade in new_trades:
                 self.db.add_trade(trade_type='cross_margin',
                                   trade_id=int(trade['id']),
@@ -350,7 +390,13 @@ def update_all_cross_margin_trades(self, limit: int = 1000):
         :return: None
         :rtype: None
         """
-        symbols_info = self.client._request_margin_api('get', 'margin/allPairs', data={})  # not built-in yet
+        client_params = {
+            'method': 'get',
+            'path': 'margin/allPairs',
+            'data': {}
+        }
+        symbols_info = self._call_binance_client('_request_margin_api', client_params)  # not built-in yet
+
         pbar = tqdm(total=len(symbols_info))
         for symbol_info in symbols_info:
             pbar.set_description(f"fetching {symbol_info['symbol']} cross margin trades")
@@ -378,10 +424,14 @@ def update_lending_redemptions(self):
             latest_time = self.db.get_last_lending_redemption_time(lending_type=lending_type) + 1
             current = 1
             while True:
-                lending_redemptions = self.client.get_lending_redemption_history(lendingType=lending_type,
-                                                                                 startTime=latest_time,
-                                                                                 current=current,
-                                                                                 size=100)
+                client_params = {
+                    'lendingType': lending_type,
+                    'startTime': latest_time,
+                    'current': current,
+                    'size': 100
+                }
+                lending_redemptions = self._call_binance_client('get_lending_redemption_history', client_params)
+
                 for li in lending_redemptions:
                     if li['status'] == 'PAID':
                         self.db.add_lending_redemption(redemption_time=li['createTime'],
@@ -416,10 +466,14 @@ def update_lending_purchases(self):
             latest_time = self.db.get_last_lending_purchase_time(lending_type=lending_type) + 1
             current = 1
             while True:
-                lending_purchases = self.client.get_lending_purchase_history(lendingType=lending_type,
-                                                                             startTime=latest_time,
-                                                                             current=current,
-                                                                             size=100)
+                client_params = {
+                    'lendingType': lending_type,
+                    'startTime': latest_time,
+                    'current': current,
+                    'size': 100
+                }
+                lending_purchases = self._call_binance_client('get_lending_purchase_history', client_params)
+
                 for li in lending_purchases:
                     if li['status'] == 'SUCCESS':
                         self.db.add_lending_purchase(purchase_id=li['purchaseId'],
@@ -455,10 +509,14 @@ def update_lending_interests(self):
             latest_time = self.db.get_last_lending_interest_time(lending_type=lending_type) + 3600 * 1000  # add 1 hour
             current = 1
             while True:
-                lending_interests = self.client.get_lending_interest_history(lendingType=lending_type,
-                                                                             startTime=latest_time,
-                                                                             current=current,
-                                                                             size=100)
+                client_params = {
+                    'lendingType': lending_type,
+                    'startTime': latest_time,
+                    'current': current,
+                    'size': 100
+                }
+                lending_interests = self._call_binance_client('get_lending_interest_history', client_params)
+
                 for li in lending_interests:
                     self.db.add_lending_interest(time=li['time'],
                                                  lending_type=li['lendingType'],
@@ -488,7 +546,7 @@ def update_spot_dusts(self):
         """
         self.db.drop_table(tables.SPOT_DUST_TABLE)
 
-        result = self.client.get_dust_log()
+        result = self._call_binance_client('get_dust_log')
         dusts = result['results']
         pbar = tqdm(total=dusts['total'])
         pbar.set_description("fetching spot dusts")
@@ -528,18 +586,21 @@ def update_spot_dividends(self, day_jump: float = 90, limit: int = 500):
         pbar = tqdm(total=math.ceil((now_millistamp - start_time) / delta_jump))
         pbar.set_description("fetching spot dividends")
         while start_time < now_millistamp:
+            # the stable working version of client.get_asset_dividend_history is not released yet,
+            # for now it has a post error, so this protected member is used in the meantime
             params = {
                 'startTime': start_time,
                 'endTime': start_time + delta_jump,
                 'limit': limit
             }
-            # the stable working version of client.get_asset_dividend_history is not released yet,
-            # for now it has a post error, so this protected member is used in the meantime
-            result = self.client._request_margin_api('get',
-                                                     'asset/assetDividend',
-                                                     True,
-                                                     data=params
-                                                     )
+            client_params = {
+                'method': 'get',
+                'path': 'asset/assetDividend',
+                'signed': True,
+                'data': params
+            }
+            result = self._call_binance_client('_request_margin_api', client_params)
+
             dividends = result['rows']
             for div in dividends:
                 self.db.add_dividend(div_id=int(div['tranId']),
@@ -579,7 +640,13 @@ def update_spot_withdraws(self, day_jump: float = 90):
         pbar = tqdm(total=math.ceil((now_millistamp - start_time) / delta_jump))
         pbar.set_description("fetching spot withdraws")
         while start_time < now_millistamp:
-            result = self.client.get_withdraw_history(startTime=start_time, endTime=start_time + delta_jump, status=6)
+            client_params = {
+                'startTime': start_time,
+                'endTime': start_time + delta_jump,
+                'status': 6
+            }
+            result = self._call_binance_client('get_withdraw_history', client_params)
+
             withdraws = result['withdrawList']
             for withdraw in withdraws:
                 self.db.add_withdraw(withdraw_id=withdraw['id'],
@@ -618,7 +685,13 @@ def update_spot_deposits(self, day_jump: float = 90):
         pbar = tqdm(total=math.ceil((now_millistamp - start_time) / delta_jump))
         pbar.set_description("fetching spot deposits")
         while start_time < now_millistamp:
-            result = self.client.get_deposit_history(startTime=start_time, endTime=start_time + delta_jump, status=1)
+            client_params = {
+                'startTime': start_time,
+                'endTime': start_time + delta_jump,
+                'status': 1
+            }
+            result = self._call_binance_client('get_deposit_history', client_params)
+
             deposits = result['depositList']
             for deposit in deposits:
                 self.db.add_deposit(tx_id=deposit['txId'],
@@ -654,7 +727,13 @@ def update_spot_symbol_trades(self, asset: str, ref_asset: str, limit: int = 100
         symbol = asset + ref_asset
         last_trade_id = self.db.get_max_trade_id(asset, ref_asset, 'spot')
         while True:
-            new_trades = self.client.get_my_trades(symbol=symbol, fromId=last_trade_id + 1, limit=limit)
+            client_params = {
+                'symbol': symbol,
+                'fromId': last_trade_id + 1,
+                'limit': limit
+            }
+            new_trades = self._call_binance_client('get_my_trades', client_params)
+
             for trade in new_trades:
                 self.db.add_trade(trade_type='spot',
                                   trade_id=int(trade['id']),
@@ -692,3 +771,35 @@ def update_all_spot_trades(self, limit: int = 1000):
                                            limit=limit)
             pbar.update()
         pbar.close()
+
+    def _call_binance_client(self, method_name: str, params: Optional[Dict] = None, retry_count: int = 0):
+        """
+        This method is used to handle rate limits: if a rate limits is breached, it will wait the necessary time
+        to call again the API.
+
+        :param method_name: name of the method binance.Client to call
+        :type method_name: str
+        :param params: parameters to pass to the above method
+        :type params: Dict
+        :param retry_count: internal use only to count the number of retry if rate limits are breached
+        :type retry_count: int
+        :return: response of binance.Client method
+        :rtype: Dict
+        """
+        if params is None:
+            params = dict()
+        if retry_count >= BinanceManager.API_MAX_RETRY:
+            raise RuntimeError(f"The API rate limits has been breached {retry_count} times")
+
+        try:
+            return getattr(self.client, method_name)(**params)
+        except BinanceAPIException as err:
+            if err.code == -1003:  # API rate Limits
+                wait_time = float(err.response.headers['Retry-After'])
+                if err.response.status_code == 418:  # ban
+                    self.logger.error(f"API calls resulted in a ban, retry in {wait_time} seconds")
+                    raise err
+                self.logger.info(f"API calls resulted in a breach of rate limits, will retry after {wait_time} seconds")
+                time.sleep(wait_time + 1)
+                return self._call_binance_client(method_name, params, retry_count + 1)
+            raise err

From e76179d611951db93081e63000f2d4a831e3110d Mon Sep 17 00:00:00 2001
From: EtWn <34377743+EtWnn@users.noreply.github.com>
Date: Mon, 29 Mar 2021 16:29:12 +0200
Subject: [PATCH 3/5] Version (#29)

* change version to v0.1.2dev

* add latest pip install

* auto version name for conf.py and setup.py
---
 BinanceWatch/__init__.py |  2 +-
 README.rst               | 14 +++++++++++---
 docs/source/conf.py      |  7 +++++--
 setup.py                 | 14 ++++++++++----
 4 files changed, 27 insertions(+), 10 deletions(-)

diff --git a/BinanceWatch/__init__.py b/BinanceWatch/__init__.py
index 863cfad..ae2aacc 100644
--- a/BinanceWatch/__init__.py
+++ b/BinanceWatch/__init__.py
@@ -1,2 +1,2 @@
-__version__ = "0.1.1"
+__version__ = "0.1.2dev"
 __author__ = 'EtWnn'
diff --git a/README.rst b/README.rst
index dbaef29..986fe43 100644
--- a/README.rst
+++ b/README.rst
@@ -1,6 +1,6 @@
-==============================
-Welcome to BinanceWatch v0.1.1
-==============================
+===================================
+Welcome to BinanceWatch v0.1.2dev
+===================================
 
 Note
 ----
@@ -62,6 +62,14 @@ permissions are needed.
 
     pip install BinanceWatch
 
+If you prefer to install the latest developments use:
+
+.. code:: bash
+
+    pip install git+https://github.com/EtWnn/BinanceWatch.git@develop
+
+Use your Binance api keys to initiate the manager:
+
 .. code:: python
 
     from BinanceWatch.BinanceManager import BinanceManager
diff --git a/docs/source/conf.py b/docs/source/conf.py
index f015d09..d697de0 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -21,8 +21,11 @@
 author = 'EtWnn'
 
 # The full version, including alpha/beta/rc tags
-release = '0.1.1'
-
+this_directory = os.path.abspath(os.path.dirname(__file__))
+about = {}
+with open(os.path.join(this_directory, '../../BinanceWatch/__init__.py'), encoding='utf-8') as f:
+    exec(f.read(), about)
+release = about['__version__']
 
 # -- General configuration ---------------------------------------------------
 
diff --git a/setup.py b/setup.py
index 25e9216..9fe30bd 100644
--- a/setup.py
+++ b/setup.py
@@ -1,12 +1,18 @@
+import os
 from setuptools import setup
-from os import path
-this_directory = path.abspath(path.dirname(__file__))
-with open(path.join(this_directory, 'README.rst'), encoding='utf-8') as f:
+
+this_directory = os.path.abspath(os.path.dirname(__file__))
+
+with open(os.path.join(this_directory, 'README.rst'), encoding='utf-8') as f:
     long_description = f.read()
 
+about = {}
+with open(os.path.join(this_directory, 'BinanceWatch/__init__.py'), encoding='utf-8') as f:
+    exec(f.read(), about)
+
 setup(
     name='BinanceWatch',
-    version='0.1.1',
+    version=about['__version__'],
     packages=['BinanceWatch',
               'tests',
               'BinanceWatch.storage',

From 95a732bd6052fcd27bae3030604f7a345bb8f88d Mon Sep 17 00:00:00 2001
From: EtWn <34377743+EtWnn@users.noreply.github.com>
Date: Tue, 30 Mar 2021 10:39:08 +0200
Subject: [PATCH 4/5] add version requirements >=0.7.9 for python-binance (#30)

---
 requirements.txt | 2 +-
 setup.py         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 293a748..ec07ed6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,7 +2,7 @@ numpy
 tqdm
 dateparser
 requests
-python-binance
+python-binance>=0.7.9
 appdirs
 sphinx
 sphinx_rtd_theme
diff --git a/setup.py b/setup.py
index 9fe30bd..2e45fd6 100644
--- a/setup.py
+++ b/setup.py
@@ -24,7 +24,7 @@
     description='Local tracker of a binance account',
     long_description=long_description,
     long_description_content_type='text/x-rst',
-    install_requires=['numpy', 'tqdm', 'dateparser', 'requests', 'python-binance', 'appdirs'],
+    install_requires=['numpy', 'tqdm', 'dateparser', 'requests', 'python-binance>=0.7.9', 'appdirs'],
     keywords='binance exchange wallet save tracking history bitcoin ethereum btc eth',
     classifiers=[
         'Intended Audience :: Developers',

From 8a28a5f722ef962112831647f96e7f8f8805188c Mon Sep 17 00:00:00 2001
From: EtWn <34377743+EtWnn@users.noreply.github.com>
Date: Tue, 30 Mar 2021 10:59:50 +0200
Subject: [PATCH 5/5] update version to 0.1.2 (#31)

---
 BinanceWatch/__init__.py | 2 +-
 README.rst               | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/BinanceWatch/__init__.py b/BinanceWatch/__init__.py
index ae2aacc..99277fe 100644
--- a/BinanceWatch/__init__.py
+++ b/BinanceWatch/__init__.py
@@ -1,2 +1,2 @@
-__version__ = "0.1.2dev"
+__version__ = "0.1.2"
 __author__ = 'EtWnn'
diff --git a/README.rst b/README.rst
index 986fe43..4889d88 100644
--- a/README.rst
+++ b/README.rst
@@ -1,5 +1,5 @@
 ===================================
-Welcome to BinanceWatch v0.1.2dev
+Welcome to BinanceWatch v0.1.2
 ===================================
 
 Note