fix not crawling finance when no event

fooltrader/api/event.py

@@ -37,7 +37,7 @@ def get_event(security_item, event_type='finance_forecast', start_date=None, end
     path = get_event_path(security_item, event_type)
 
     if os.path.exists(path):
-        df = pd_utils.pd_read_csv(path, index=index)
+        df = pd_utils.pd_read_csv(path, index=index, generate_id=True)
         df = df_for_date_range(df, start_date=start_date, end_date=end_date)
     else:
         df = pd.DataFrame()

fooltrader/api/technical.py

@@ -92,6 +92,7 @@ def get_security_list(security_type='stock', exchanges=None, start_code=None, en
         df = df_for_date_range(df, start_date=start_list_date)
 
         df = df.set_index(df['code'], drop=False)
+        df = df.sort_index()
 
         if codes:
             df = df.loc[codes]

fooltrader/datamanager/china_stock_manager.py

@@ -7,8 +7,8 @@
 import pandas as pd
 
 from fooltrader import settings
-from fooltrader.api import event
-from fooltrader.api.fundamental import get_balance_sheet_items, get_income_statement_items, get_cash_flow_statement_items
+from fooltrader.api.fundamental import get_balance_sheet_items, get_income_statement_items, \
+    get_cash_flow_statement_items
 from fooltrader.api.technical import get_security_list, get_latest_download_trading_date, get_trading_dates, \
     get_available_tick_dates, get_kdata
 from fooltrader.contract.files_contract import get_balance_sheet_path, get_income_statement_path, \
@@ -62,10 +62,10 @@ def crawl_finance_data(start_code=STOCK_START_CODE, end_code=STOCK_END_CODE):
 
                 if current_report_period != current_items[-1]['reportPeriod']:
                     # 报告出来了
-                    df = event.get_finance_report_event(security_item, index='reportPeriod')
-                    if current_report_period in df.index:
-                        process_crawl(StockFinanceSpider, {"security_item": security_item,
-                                                           "report_type": "balance_sheet"})
+                    # df = event.get_finance_report_event(security_item, index='reportPeriod')
+                    # if current_report_period in df.index:
+                    process_crawl(StockFinanceSpider, {"security_item": security_item,
+                                                       "report_type": "balance_sheet"})
 
             # 利润表
             path = get_income_statement_path(security_item)
@@ -77,10 +77,10 @@ def crawl_finance_data(start_code=STOCK_START_CODE, end_code=STOCK_END_CODE):
                 # 当前报告期还没抓取
                 if current_report_period != current_items[-1]['reportPeriod']:
                     # 报告出来了
-                    df = event.get_finance_report_event(security_item, index='reportPeriod')
-                    if current_report_period in df.index:
-                        process_crawl(StockFinanceSpider, {"security_item": security_item,
-                                                           "report_type": "income_statement"})
+                    # df = event.get_finance_report_event(security_item, index='reportPeriod')
+                    # if current_report_period in df.index:
+                    process_crawl(StockFinanceSpider, {"security_item": security_item,
+                                                       "report_type": "income_statement"})
 
             # 现金流量表
             path = get_cash_flow_statement_path(security_item)
@@ -92,10 +92,10 @@ def crawl_finance_data(start_code=STOCK_START_CODE, end_code=STOCK_END_CODE):
                 # 当前报告期还没抓取
                 if current_report_period != current_items[-1]['reportPeriod']:
                     # 报告出来了
-                    df = event.get_finance_report_event(security_item, index='reportPeriod')
-                    if current_report_period in df.index:
-                        process_crawl(StockFinanceSpider, {"security_item": security_item,
-                                                           "report_type": "cash_flow"})
+                    # df = event.get_finance_report_event(security_item, index='reportPeriod')
+                    # if current_report_period in df.index:
+                    process_crawl(StockFinanceSpider, {"security_item": security_item,
+                                                       "report_type": "cash_flow"})
         except Exception as e:
             logger.exception(e)
 
@@ -163,7 +163,8 @@ def crawl_stock_quote(start_code=STOCK_START_CODE, end_code=STOCK_END_CODE, craw
                 logger.info("{} {} kdata from sina is ok".format(security_item['code'], fuquan))
 
         # 抓取tick
-        if crawl_tick:
+        # FIXME:新浪该服务已不可用
+        if crawl_tick and False:
             tick_dates = {x for x in base_dates if x >= settings.START_TICK_DATE}
             diff_dates = tick_dates - set(get_available_tick_dates(security_item))
 

fooltrader/spiders/chinastock/stock_kdata_sina_spider.py

@@ -145,6 +145,10 @@ def merge_to_current_kdata(security_item, df, fuquan='bfq'):
         df1 = df1.sort_index()
 
         the_path = files_contract.get_kdata_path(security_item, source='sina', fuquan=fuquan)
+        if fuquan == 'hfq':
+            df1 = df1.loc[:, data_contract.KDATA_COLUMN_SINA_FQ]
+        else:
+            df1 = df1.loc[:, data_contract.KDATA_COLUMN_SINA]
         df1.to_csv(the_path, index=False)
 
     @staticmethod

fooltrader/utils/pd_utils.py

@@ -3,9 +3,6 @@
 
 import pandas as pd
 
-from fooltrader.settings import TIME_FORMAT_SEC
-from fooltrader.utils.utils import to_time_str
-
 logger = logging.getLogger(__name__)
 
 
@@ -52,9 +49,7 @@ def pd_read_csv(csv_path, converters=None, index='timestamp', generate_id=False)
     if not df.empty:
         # generate id if need
         if generate_id and 'id' not in df.columns and 'securityId' in df.columns and 'timestamp' in df.columns:
-            timestamp_str = df.timestamp.apply(lambda x: to_time_str(x, time_fmt=TIME_FORMAT_SEC))
-
-            df['id'] = df['securityId'] + '_' + timestamp_str
+            df['id'] = df['securityId'] + '_' + df['timestamp']
 
         df = df.set_index(df[index], drop=False)