From 30406e511781feb9e78b4c00126976da11cf7561 Mon Sep 17 00:00:00 2001 From: Chris Morgan Date: Thu, 17 Dec 2015 08:59:48 +0000 Subject: [PATCH] autopep8 -i -r -a . --- examples/introduction/asimpletradingrule.py | 49 +- examples/introduction/prebakedsystems.py | 10 +- examples/introduction/simplesystem.py | 112 ++--- setup.py | 24 +- syscore/accounting.py | 446 +++++++++--------- syscore/algos.py | 60 +-- syscore/dateutils.py | 84 ++-- syscore/fileutils.py | 49 +- syscore/genutils.py | 30 +- syscore/objects.py | 77 +-- syscore/pdutils.py | 234 ++++----- syscore/tests/test_Accounting.py | 94 ++-- syscore/tests/test_algos.py | 67 +-- syscore/tests/test_dateutils.py | 14 +- syscore/tests/test_pdutils.py | 38 +- sysdata/configdata.py | 80 ++-- sysdata/csvdata.py | 160 +++---- sysdata/data.py | 169 ++++--- sysdata/futuresdata.py | 33 +- sysdata/quandl.py | 1 - systems/account.py | 77 +-- systems/basesystem.py | 373 +++++++-------- systems/defaults.py | 11 +- systems/forecast_combine.py | 270 ++++++----- systems/forecast_scale_cap.py | 187 ++++---- systems/forecasting.py | 408 ++++++++-------- systems/futures/rawdata.py | 154 +++--- systems/portfolio.py | 251 +++++----- systems/positionsizing.py | 349 +++++++------- systems/provided/example/rules.py | 73 +-- systems/provided/example/simplesystem.py | 10 +- .../provided/futures_chapter15/basesystem.py | 35 +- systems/provided/futures_chapter15/rules.py | 54 ++- systems/rawdata.py | 201 ++++---- systems/stage.py | 25 +- systems/tests/test_base_systems.py | 67 +-- systems/tests/test_forecasts.py | 222 ++++----- systems/tests/testdata.py | 75 +-- 38 files changed, 2407 insertions(+), 2266 deletions(-) diff --git a/examples/introduction/asimpletradingrule.py b/examples/introduction/asimpletradingrule.py index a3e053db74..b23268c1a5 100644 --- a/examples/introduction/asimpletradingrule.py +++ b/examples/introduction/asimpletradingrule.py @@ -4,7 +4,7 @@ """ -## Get some data +# Get some data from sysdata.csvdata import csvFuturesData @@ -13,10 +13,10 @@ We can get data from various places; however for now we're going to use prepackaged 'legacy' data stored in csv files - + """ -data=csvFuturesData() +data = csvFuturesData() print(data) @@ -66,29 +66,30 @@ from syscore.algos import robust_vol_calc from syscore.pdutils import divide_df_single_column + def calc_ewmac_forecast(price, Lfast, Lslow=None): - - """ Calculate the ewmac trading fule forecast, given a price and EWMA speeds Lfast, Lslow and vol_lookback - + Assumes that 'price' is daily data """ - ## price: This is the stitched price series - ## We can't use the price of the contract we're trading, or the volatility will be jumpy - ## And we'll miss out on the rolldown. See http://qoppac.blogspot.co.uk/2015/05/systems-building-futures-rolling.html + # price: This is the stitched price series + # We can't use the price of the contract we're trading, or the volatility will be jumpy + # And we'll miss out on the rolldown. See + # http://qoppac.blogspot.co.uk/2015/05/systems-building-futures-rolling.html if Lslow is None: - Lslow=4*Lfast - - ## We don't need to calculate the decay parameter, just use the span directly - - fast_ewma=pd.ewma(price, span=Lfast) - slow_ewma=pd.ewma(price, span=Lslow) - raw_ewmac=fast_ewma - slow_ewma - - vol=robust_vol_calc(price.diff()) - + Lslow = 4 * Lfast + + # We don't need to calculate the decay parameter, just use the span + # directly + + fast_ewma = pd.ewma(price, span=Lfast) + slow_ewma = pd.ewma(price, span=Lslow) + raw_ewmac = fast_ewma - slow_ewma + + vol = robust_vol_calc(price.diff()) + return divide_df_single_column(raw_ewmac, vol) """ @@ -96,9 +97,9 @@ def calc_ewmac_forecast(price, Lfast, Lslow=None): (this isn't properly scaled at this stage of course) """ -instrument_code='EDOLLAR' -price=data.get_instrument_price(instrument_code) -ewmac=calc_ewmac_forecast(price, 32, 128) +instrument_code = 'EDOLLAR' +price = data.get_instrument_price(instrument_code) +ewmac = calc_ewmac_forecast(price, 32, 128) print(ewmac.tail(5)) from matplotlib.pyplot import show @@ -110,8 +111,6 @@ def calc_ewmac_forecast(price, Lfast, Lslow=None): """ from syscore.accounting import pandl -account=pandl(price, forecast=ewmac) +account = pandl(price, forecast=ewmac) account.curve().plot() show() - - diff --git a/examples/introduction/prebakedsystems.py b/examples/introduction/prebakedsystems.py index 61e6960bbc..6384fe5614 100644 --- a/examples/introduction/prebakedsystems.py +++ b/examples/introduction/prebakedsystems.py @@ -1,6 +1,6 @@ from systems.provided.example.simplesystem import simplesystem -my_system=simplesystem() +my_system = simplesystem() print(my_system) print(my_system.portfolio.get_notional_position("EDOLLAR").tail(5)) @@ -11,9 +11,9 @@ Now loading config and data """ -my_config=Config("systems.provided.example.simplesystemconfig.yaml") -my_data=csvFuturesData() -my_system=simplesystem(config=my_config, data=my_data) +my_config = Config("systems.provided.example.simplesystemconfig.yaml") +my_data = csvFuturesData() +my_system = simplesystem(config=my_config, data=my_data) print(my_system.portfolio.get_notional_position("EDOLLAR").tail(5)) @@ -23,6 +23,6 @@ from systems.provided.futures_chapter15.basesystem import futures_system -system=futures_system() +system = futures_system() print(system.portfolio.get_notional_position("EUROSTX").tail(5)) diff --git a/examples/introduction/simplesystem.py b/examples/introduction/simplesystem.py index dcbcefe9e6..18f1c5ccfd 100644 --- a/examples/introduction/simplesystem.py +++ b/examples/introduction/simplesystem.py @@ -4,7 +4,7 @@ We got some data and created a trading rule """ from sysdata.csvdata import csvFuturesData -data=csvFuturesData() +data = csvFuturesData() from systems.provided.example.rules import ewmac_forecast_with_defaults as ewmac @@ -21,36 +21,34 @@ """ - from systems.forecasting import Rules """ We can create rules in a number of different ways -Note that to make our rule work it needs to have +Note that to make our rule work it needs to have """ -my_rules=Rules(ewmac) +my_rules = Rules(ewmac) print(my_rules.trading_rules()) -my_rules=Rules(dict(ewmac=ewmac)) +my_rules = Rules(dict(ewmac=ewmac)) print(my_rules.trading_rules()) from systems.basesystem import System -my_system=System([my_rules], data) +my_system = System([my_rules], data) print(my_system) print(my_system.rules.get_raw_forecast("EDOLLAR", "ewmac").tail(5)) - """ Define a TradingRule """ from systems.forecasting import TradingRule -ewmac_rule=TradingRule(ewmac) -my_rules=Rules(dict(ewmac=ewmac_rule)) +ewmac_rule = TradingRule(ewmac) +my_rules = Rules(dict(ewmac=ewmac_rule)) ewmac_rule @@ -58,54 +56,56 @@ ... or two... """ -ewmac_8=TradingRule((ewmac, [], dict(Lfast=8, Lslow=32))) -ewmac_32=TradingRule(dict(function=ewmac, other_args=dict(Lfast=32, Lslow=128))) -my_rules=Rules(dict(ewmac8=ewmac_8, ewmac32=ewmac_32)) +ewmac_8 = TradingRule((ewmac, [], dict(Lfast=8, Lslow=32))) +ewmac_32 = TradingRule( + dict(function=ewmac, other_args=dict(Lfast=32, Lslow=128))) +my_rules = Rules(dict(ewmac8=ewmac_8, ewmac32=ewmac_32)) print(my_rules.trading_rules()['ewmac32']) -my_system=System([my_rules], data) +my_system = System([my_rules], data) my_system.rules.get_raw_forecast("EDOLLAR", "ewmac32").tail(5) from sysdata.configdata import Config -my_config=Config() +my_config = Config() my_config -empty_rules=Rules() -my_config.trading_rules=dict(ewmac8=ewmac_8, ewmac32=ewmac_32) -my_system=System([empty_rules], data, my_config) +empty_rules = Rules() +my_config.trading_rules = dict(ewmac8=ewmac_8, ewmac32=ewmac_32) +my_system = System([empty_rules], data, my_config) my_system.rules.get_raw_forecast("EDOLLAR", "ewmac32").tail(5) from systems.forecast_scale_cap import ForecastScaleCapFixed -my_config.forecast_scalars=dict(ewmac8=5.3, ewmac32=2.65) -fcs=ForecastScaleCapFixed() -my_system=System([fcs, my_rules], data, my_config) -print(my_system.forecastScaleCap.get_capped_forecast("EDOLLAR", "ewmac32").tail(5)) +my_config.forecast_scalars = dict(ewmac8=5.3, ewmac32=2.65) +fcs = ForecastScaleCapFixed() +my_system = System([fcs, my_rules], data, my_config) +print(my_system.forecastScaleCap.get_capped_forecast( + "EDOLLAR", "ewmac32").tail(5)) """ combine some rules """ from systems.forecast_combine import ForecastCombineFixed -#forecast_weights=dict(ewmac8=0.5, ewmac32=0.5), forecast_div_multiplier=1.1 -combiner=ForecastCombineFixed() -my_system=System([fcs, my_rules, combiner], data, my_config) +# forecast_weights=dict(ewmac8=0.5, ewmac32=0.5), forecast_div_multiplier=1.1 +combiner = ForecastCombineFixed() +my_system = System([fcs, my_rules, combiner], data, my_config) print(my_system.combForecast.get_combined_forecast("EDOLLAR").tail(5)) -my_config.forecast_weights=dict(ewmac8=0.5, ewmac32=0.5) -my_config.forecast_div_multiplier=1.1 -my_system=System([fcs, empty_rules, combiner], data, my_config) +my_config.forecast_weights = dict(ewmac8=0.5, ewmac32=0.5) +my_config.forecast_div_multiplier = 1.1 +my_system = System([fcs, empty_rules, combiner], data, my_config) my_system.combForecast.get_combined_forecast("EDOLLAR").tail(5) -## size positions +# size positions from systems.positionsizing import PositionSizing -possizer=PositionSizing() -my_config.percentage_vol_target=25 -my_config.notional_trading_capital=500000 -my_config.base_currency="GBP" +possizer = PositionSizing() +my_config.percentage_vol_target = 25 +my_config.notional_trading_capital = 500000 +my_config.base_currency = "GBP" -my_system=System([ fcs, my_rules, combiner, possizer], data, my_config) +my_system = System([fcs, my_rules, combiner, possizer], data, my_config) print(my_system.positionSize.get_price_volatility("EDOLLAR").tail(5)) print(my_system.positionSize.get_block_value("EDOLLAR").tail(5)) @@ -115,13 +115,14 @@ print(my_system.positionSize.get_daily_cash_vol_target()) print(my_system.positionSize.get_subsystem_position("EDOLLAR").tail(5)) -## portfolio +# portfolio from systems.portfolio import PortfoliosFixed -portfolio=PortfoliosFixed() -my_config.instrument_weights=dict(US10=.1, EDOLLAR=.4, CORN=.3, SP500=.2) -my_config.instrument_div_multiplier=1.5 +portfolio = PortfoliosFixed() +my_config.instrument_weights = dict(US10=.1, EDOLLAR=.4, CORN=.3, SP500=.2) +my_config.instrument_div_multiplier = 1.5 -my_system=System([ fcs, my_rules, combiner, possizer, portfolio], data, my_config) +my_system = System([fcs, my_rules, combiner, possizer, + portfolio], data, my_config) print(my_system.portfolio.get_notional_position("EDOLLAR").tail(5)) @@ -130,35 +131,38 @@ """ from systems.account import Account -my_account=Account() -my_system=System([ fcs, my_rules, combiner, possizer, portfolio, my_account], data, my_config) -profits=my_system.account.portfolio() +my_account = Account() +my_system = System([fcs, my_rules, combiner, possizer, + portfolio, my_account], data, my_config) +profits = my_system.account.portfolio() profits.stats() """ -Another approach is to create a config object +Another approach is to create a config object """ -my_config=Config(dict(trading_rules=dict(ewmac8=ewmac_8, ewmac32=ewmac_32), - instrument_weights=dict(US10=.1, EDOLLAR=.4, CORN=.3, SP500=.2), - instrument_div_multiplier=1.5, forecast_scalars=dict(ewmac8=5.3, ewmac32=2.65), - forecast_weights=dict(ewmac8=0.5, ewmac32=0.5), forecast_div_multiplier=1.1, - percentage_vol_target=25.00, notional_trading_capital=500000, base_currency="GBP")) +my_config = Config(dict(trading_rules=dict(ewmac8=ewmac_8, ewmac32=ewmac_32), + instrument_weights=dict( + US10=.1, EDOLLAR=.4, CORN=.3, SP500=.2), + instrument_div_multiplier=1.5, forecast_scalars=dict(ewmac8=5.3, ewmac32=2.65), + forecast_weights=dict(ewmac8=0.5, ewmac32=0.5), forecast_div_multiplier=1.1, + percentage_vol_target=25.00, notional_trading_capital=500000, base_currency="GBP")) print(my_config) -my_system=System([Account(), PortfoliosFixed(), PositionSizing(), ForecastCombineFixed(), ForecastScaleCapFixed(), Rules() -], data, my_config) +my_system = System([Account(), PortfoliosFixed(), PositionSizing(), ForecastCombineFixed(), ForecastScaleCapFixed(), Rules() + ], data, my_config) print(my_system.portfolio.get_notional_position("EDOLLAR").tail(5)) """ ... or to import one """ -my_config=Config("systems.provided.example.simplesystemconfig.yaml") +my_config = Config("systems.provided.example.simplesystemconfig.yaml") print(my_config) -my_system=System([Account(), PortfoliosFixed(), PositionSizing(), ForecastCombineFixed(), ForecastScaleCapFixed(), Rules() -], data, my_config) +my_system = System([Account(), PortfoliosFixed(), PositionSizing(), ForecastCombineFixed(), ForecastScaleCapFixed(), Rules() + ], data, my_config) print(my_system.rules.get_raw_forecast("EDOLLAR", "ewmac32").tail(5)) print(my_system.rules.get_raw_forecast("EDOLLAR", "ewmac8").tail(5)) -print(my_system.forecastScaleCap.get_capped_forecast("EDOLLAR", "ewmac32").tail(5)) +print(my_system.forecastScaleCap.get_capped_forecast( + "EDOLLAR", "ewmac32").tail(5)) print(my_system.forecastScaleCap.get_forecast_scalar("EDOLLAR", "ewmac32")) print(my_system.combForecast.get_combined_forecast("EDOLLAR").tail(5)) print(my_system.combForecast.get_forecast_weights("EDOLLAR").tail(5)) @@ -166,5 +170,3 @@ print(my_system.positionSize.get_subsystem_position("EDOLLAR").tail(5)) print(my_system.portfolio.get_notional_position("EDOLLAR").tail(5)) - - diff --git a/setup.py b/setup.py index 4c524cc35f..e18a6b1533 100644 --- a/setup.py +++ b/setup.py @@ -2,22 +2,24 @@ from setuptools import setup # Utility function to read the README file. + + def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() setup( - name = "pysystemtrade", - version = "0.0.1", - author = "Robert Carver", - description = ("Python framework for running systems as in Robert Carver's book Systematic Trading" - " (www.systematictrading.org)"), - license = "GNU GPL v3", - keywords = "systematic trading interactive brokers", - url = "qoppac.blogspot.com/pysystemtrade.html", - packages=['examples','syscore','sysdata','systems','syssims'], + name="pysystemtrade", + version="0.0.1", + author="Robert Carver", + description=("Python framework for running systems as in Robert Carver's book Systematic Trading" + " (www.systematictrading.org)"), + license="GNU GPL v3", + keywords="systematic trading interactive brokers", + url="qoppac.blogspot.com/pysystemtrade.html", + packages=['examples', 'syscore', 'sysdata', 'systems', 'syssims'], long_description=read('README.md'), install_requires=["pandas >= 0.17.0", "numpy >= 1.10.1", "python >= 3.4.3", "matlotplib > 1.4.3", - "yaml > 3.11"], + "yaml > 3.11"], extras_require=dict(), include_package_data=True -) \ No newline at end of file +) diff --git a/syscore/accounting.py b/syscore/accounting.py index 9a41ec495f..98d8ef3e86 100644 --- a/syscore/accounting.py +++ b/syscore/accounting.py @@ -17,50 +17,51 @@ """ some defaults """ -CAPITAL=10000000.0 -ANN_RISK_TARGET=0.16 -DAILY_CAPITAL=CAPITAL*ANN_RISK_TARGET/ROOT_BDAYS_INYEAR +CAPITAL = 10000000.0 +ANN_RISK_TARGET = 0.16 +DAILY_CAPITAL = CAPITAL * ANN_RISK_TARGET / ROOT_BDAYS_INYEAR -def pandl(price=None, trades=None, marktomarket=True, positions=None, delayfill=True, roundpositions=False, - price_change_volatility=None, forecast=None, fx=None, value_of_price_point=1.0, + +def pandl(price=None, trades=None, marktomarket=True, positions=None, delayfill=True, roundpositions=False, + price_change_volatility=None, forecast=None, fx=None, value_of_price_point=1.0, return_all=False): """ Calculate pandl for an individual position - + If marktomarket=True, and trades is provided, calculate pandl both at open/close and mark to market in between - - If trades is not provided, work out using positions. + + If trades is not provided, work out using positions. If delayfill is True, assume we get filled at the next price after the trade - - If roundpositions is True when working out trades from positions, then round; otherwise assume we trade fractional lots - + + If roundpositions is True when working out trades from positions, then round; otherwise assume we trade fractional lots + If positions are not provided, work out position using forecast and volatility (this will be for an arbitrary daily risk target) - + If volatility is not provided, work out from price - + If fx is not provided, assume fx rate is 1.0 and work out p&l in currency of instrument - + If value_of_price_point is not provided, assume is 1.0 (block size is value of 1 price point, eg 100 if you're buying 100 shares for one instrument block) - + :param price: price series :type price: Tx1 pd.DataFrame :param trades: set of trades done - :type trades: Tx1 pd.DataFrame or None + :type trades: Tx1 pd.DataFrame or None :param marktomarket: Should we mark to market, or just use traded prices? - :type marktomarket: bool + :type marktomarket: bool :param positions: series of positions - :type positions: Tx1 pd.DataFrame or None + :type positions: Tx1 pd.DataFrame or None :param delayfill: If calculating trades, should we round positions first? - :type delayfill: bool + :type delayfill: bool :param roundpositions: If calculating trades, should we round positions first? - :type roundpositions: bool + :type roundpositions: bool :param price_change_volatility: series of volatility estimates, used for calculation positions :type price_change_volatility: Tx1 pd.DataFrame or None @@ -75,88 +76,95 @@ def pandl(price=None, trades=None, marktomarket=True, positions=None, delayfill= :type value_of_price_point: float :param roundpositions: If calculating trades, should we round positions first? - :type roundpositions: bool + :type roundpositions: bool + - :returns: if return_all : 4- Tuple (positions, trades, instr_ccy_returns, base_ccy_returns) all Tx1 pd.DataFrames is "": Tx1 accountCurve - - + + """ if price is None: raise Exception("Can't work p&l without price") - + if fx is None: - ## assume it's 1.0 - fx=pd.Series([1.0]*len(price.index), index=price.index).to_frame("fx") - + # assume it's 1.0 + fx = pd.Series([1.0] * len(price.index), + index=price.index).to_frame("fx") + if trades is None: - trades=get_trades_from_positions(price, positions, delayfill, roundpositions, price_change_volatility, forecast, fx, value_of_price_point) + trades = get_trades_from_positions( + price, positions, delayfill, roundpositions, price_change_volatility, forecast, fx, value_of_price_point) if marktomarket: - ### want to have both kinds of price - prices_to_use=pd.concat([price, trades.fill_price], axis=1, join='outer') - - ## Where no fill price available, use price - prices_to_use=prices_to_use.fillna(axis=1, method="ffill") - - prices_to_use=prices_to_use.fill_price.to_frame("price") - - ## alight trades - - trades_to_use=trades.reindex(prices_to_use.index, fill_value=0.0).trades.to_frame("trades") + # want to have both kinds of price + prices_to_use = pd.concat( + [price, trades.fill_price], axis=1, join='outer') + + # Where no fill price available, use price + prices_to_use = prices_to_use.fillna(axis=1, method="ffill") + + prices_to_use = prices_to_use.fill_price.to_frame("price") + + # alight trades + + trades_to_use = trades.reindex( + prices_to_use.index, fill_value=0.0).trades.to_frame("trades") else: - ### only calculate p&l on trades, using fills - trades_to_use=trades.trades.to_frame("trades") - prices_to_use=trades.fill_price.to_frame("price").ffill() - - cum_trades=trades_to_use.cumsum().ffill() - price_returns=prices_to_use.diff() - - instr_ccy_returns=multiply_df_single_column(cum_trades.shift(1), price_returns)*value_of_price_point - fx=fx.reindex(trades_to_use.index, method="ffill") - base_ccy_returns=multiply_df_single_column(instr_ccy_returns, fx) - - instr_ccy_returns.columns=["pandl_ccy"] - base_ccy_returns.columns=["pandl_base"] - cum_trades.columns=["cum_trades"] - + # only calculate p&l on trades, using fills + trades_to_use = trades.trades.to_frame("trades") + prices_to_use = trades.fill_price.to_frame("price").ffill() + + cum_trades = trades_to_use.cumsum().ffill() + price_returns = prices_to_use.diff() + + instr_ccy_returns = multiply_df_single_column( + cum_trades.shift(1), price_returns) * value_of_price_point + fx = fx.reindex(trades_to_use.index, method="ffill") + base_ccy_returns = multiply_df_single_column(instr_ccy_returns, fx) + + instr_ccy_returns.columns = ["pandl_ccy"] + base_ccy_returns.columns = ["pandl_base"] + cum_trades.columns = ["cum_trades"] + if return_all: return (cum_trades, trades, instr_ccy_returns, base_ccy_returns) else: if positions is None: - ## arbitrary, return in % terms - capital=CAPITAL + # arbitrary, return in % terms + capital = CAPITAL else: - ## return in money terms - capital=None - + # return in money terms + capital = None + return accountCurve(base_ccy_returns, capital=CAPITAL) - -def get_trades_from_positions(price, positions, delayfill, roundpositions, price_change_volatility, forecast, fx, value_of_price_point): + + +def get_trades_from_positions(price, positions, delayfill, roundpositions, + price_change_volatility, forecast, fx, value_of_price_point): """ Work out trades implied by a series of positions If delayfill is True, assume we get filled at the next price after the trade - If roundpositions is True when working out trades from positions, then round; otherwise assume we trade fractional lots + If roundpositions is True when working out trades from positions, then round; otherwise assume we trade fractional lots If positions are not provided, work out position using forecast and volatility (this will be for an arbitrary daily risk target) - + If volatility is not provided, work out from price - - + + :param price: price series :type price: Tx1 pd.DataFrame :param positions: series of positions - :type positions: Tx1 pd.DataFrame or None + :type positions: Tx1 pd.DataFrame or None :param delayfill: If calculating trades, should we round positions first? - :type delayfill: bool + :type delayfill: bool :param roundpositions: If calculating trades, should we round positions first? - :type roundpositions: bool + :type roundpositions: bool :param price_change_volatility: series of volatility estimates, used for calculation positions :type price_change_volatility: Tx1 pd.DataFrame or None @@ -170,108 +178,116 @@ def get_trades_from_positions(price, positions, delayfill, roundpositions, price :param block_size: value of one movement in price :type block_size: float - + :returns: Tx1 pd dataframe of trades - + """ - + if positions is None: - positions=get_positions_from_forecasts(price, price_change_volatility, forecast, fx, value_of_price_point) - + positions = get_positions_from_forecasts( + price, price_change_volatility, forecast, fx, value_of_price_point) + if roundpositions: - ## round to whole positions - round_positions=positions.round() + # round to whole positions + round_positions = positions.round() else: - round_positions=copy(positions) + round_positions = copy(positions) - ## deal with edge cases where we don't have a zero position initially, or leading nans - first_row=pd.DataFrame([0.0], index=[round_positions.index[0]-BDay(1)]) - first_row.columns=round_positions.columns - round_positions=pd.concat([first_row, round_positions], axis=0) - round_positions=round_positions.ffill() + # deal with edge cases where we don't have a zero position initially, or + # leading nans + first_row = pd.DataFrame([0.0], index=[round_positions.index[0] - BDay(1)]) + first_row.columns = round_positions.columns + round_positions = pd.concat([first_row, round_positions], axis=0) + round_positions = round_positions.ffill() - trades=round_positions.diff() + trades = round_positions.diff() if delayfill: - ## fill will happen one day after we generated trade (being conservative) - trades.index=trades.index+pd.DateOffset(1) - - ## put prices on to correct timestamp - ans=pd.concat([trades, price], axis=1, join='outer') - ans.columns=['trades', 'fill_price'] - - - ## fill will happen at next valid price if it happens to be missing - - ans.fill_price=ans.fill_price.fillna(method="bfill") - - ## remove zeros (turns into nans) - ans=ans[ans.trades!=0.0] - ans=ans[np.isfinite(ans.trades)] - + # fill will happen one day after we generated trade (being + # conservative) + trades.index = trades.index + pd.DateOffset(1) + + # put prices on to correct timestamp + ans = pd.concat([trades, price], axis=1, join='outer') + ans.columns = ['trades', 'fill_price'] + + # fill will happen at next valid price if it happens to be missing + + ans.fill_price = ans.fill_price.fillna(method="bfill") + + # remove zeros (turns into nans) + ans = ans[ans.trades != 0.0] + ans = ans[np.isfinite(ans.trades)] + return ans - - - -def get_positions_from_forecasts(price, price_change_volatility, forecast, fx, value_of_price_point, **kwargs): + + +def get_positions_from_forecasts( + price, price_change_volatility, forecast, fx, value_of_price_point, **kwargs): """ Work out position using forecast, volatility, fx, value_of_price_point (this will be for an arbitrary daily risk target) - + If volatility is not provided, work out from price (uses a standard method so may differ from precise system p&l) - + :param price: price series :type price: Tx1 pd.DataFrame - + :param price_change_volatility: series of volatility estimates. NOT % volatility, price difference vol :type price_change_volatility: Tx1 pd.DataFrame or None :param forecast: series of forecasts, needed to work out positions - :type forecast: Tx1 pd.DataFrame + :type forecast: Tx1 pd.DataFrame :param fx: series of fx rates from instrument currency to base currency, to work out p&l in base currency - :type fx: Tx1 pd.DataFrame + :type fx: Tx1 pd.DataFrame :param value_of_price_point: value of one unit movement in price :type value_of_price_point: float **kwargs: passed to vol calculation - + :returns: Tx1 pd dataframe of positions - + """ if forecast is None: - raise Exception("If you don't provide a series of trades or positions, I need a forecast") + raise Exception( + "If you don't provide a series of trades or positions, I need a forecast") if price_change_volatility is None: - price_change_volatility=robust_vol_calc(price.diff(), **kwargs) + price_change_volatility = robust_vol_calc(price.diff(), **kwargs) """ Herein the proof why this position calculation is correct (see chapters 5-11 of 'systematic trading' book) - + Position = forecast x instrument weight x instrument_div_mult x vol_scalar / 10.0 = forecast x instrument weight x instrument_div_mult x daily cash vol target / (10.0 x instr value volatility) = forecast x instrument weight x instrument_div_mult x daily cash vol target / (10.0 x instr ccy volatility x fx rate) = forecast x instrument weight x instrument_div_mult x daily cash vol target / (10.0 x block value x % price volatility x fx rate) = forecast x instrument weight x instrument_div_mult x daily cash vol target / (10.0 x underlying price x 0.01 x value of price move x 100 x price diff volatility/(underlying price) x fx rate) = forecast x instrument weight x instrument_div_mult x daily cash vol target / (10.0 x x value of price move x price change volatility x fx rate) - + Making some arbitrary assumptions (one instrument, 100% of capital, daily target DAILY_CAPITAL): - + = forecast x 1.0 x 1.0 x DAILY_CAPITAL / (10.0 x x value of price move x price diff volatility x fx rate) = forecast x multiplier / (value of price move x price change volatility x fx rate) - + """ - - multiplier=DAILY_CAPITAL*1.0*1.0/10.0 - fx=fx.reindex(price_change_volatility.index, method="ffill") - denominator=value_of_price_point * multiply_df_single_column(price_change_volatility,fx, ffill=(False, True)) - - position = divide_df_single_column(forecast * multiplier, denominator, ffill=(True, True)) - position.columns=['position'] + + multiplier = DAILY_CAPITAL * 1.0 * 1.0 / 10.0 + fx = fx.reindex(price_change_volatility.index, method="ffill") + denominator = value_of_price_point * \ + multiply_df_single_column( + price_change_volatility, fx, ffill=(False, True)) + + position = divide_df_single_column( + forecast * multiplier, denominator, ffill=(True, True)) + position.columns = ['position'] return position + class accountCurve(pd.DataFrame): + def __init__(self, returns=None, capital=None, **kwargs): """ Create an account curve; from which many lovely statistics can be gathered @@ -282,7 +298,7 @@ def __init__(self, returns=None, capital=None, **kwargs): :param returns: series of returns :type price: Tx1 pd.DataFrame - + :param capital: scaling factor for accounting :type capital: None float : fixed scaling will be applied @@ -291,40 +307,39 @@ def __init__(self, returns=None, capital=None, **kwargs): """ if returns is None: - returns=pandl(**kwargs) - + returns = pandl(**kwargs) + if capital is not None: - if type(capital) is float: - returns=returns/capital + if isinstance(capital, float): + returns = returns / capital else: - capital=capital.reindex(returns.index, method="ffill") - returns=divide_df_single_column(returns, capital) - + capital = capital.reindex(returns.index, method="ffill") + returns = divide_df_single_column(returns, capital) + super(accountCurve, self).__init__(returns) def daily(self): - ## we cache this since it's used so much - - if hasattr(self,"_daily_series"): + # we cache this since it's used so much + + if hasattr(self, "_daily_series"): return self._daily_series else: - daily_returns=self.resample("1B", how="sum") + daily_returns = self.resample("1B", how="sum") setattr(self, "_daily_series", daily_returns) return daily_returns def curve(self): - ## we cache this since it's used so much - if hasattr(self,"_curve"): + # we cache this since it's used so much + if hasattr(self, "_curve"): return self._curve else: - curve=self.cumsum().ffill() + curve = self.cumsum().ffill() setattr(self, "_curve", curve) return curve def weekly(self): return self.resample("W", how="sum") - def monthly(self): return self.resample("MS", how="sum") @@ -332,133 +347,132 @@ def annual(self): return self.resample("A", how="sum") def ann_daily_mean(self): - x=self.daily() - avg=float(x.mean()) - - return avg*BUSINESS_DAYS_IN_YEAR - + x = self.daily() + avg = float(x.mean()) + + return avg * BUSINESS_DAYS_IN_YEAR + def ann_daily_std(self): - x=self.daily() - daily_std=float(x.std()) - - return daily_std*ROOT_BDAYS_INYEAR - + x = self.daily() + daily_std = float(x.std()) + + return daily_std * ROOT_BDAYS_INYEAR + def sharpe(self): - mean_return=self.ann_daily_mean() - vol=self.ann_daily_std() - - return mean_return/vol - + mean_return = self.ann_daily_mean() + vol = self.ann_daily_std() + + return mean_return / vol + def drawdown(self): - x=self.curve() + x = self.curve() return drawdown(x) - + def avg_drawdown(self): - dd=self.drawdown() + dd = self.drawdown() return np.nanmean(dd.values) def worst_drawdown(self): - dd=self.drawdown() + dd = self.drawdown() return np.nanmin(dd.values) - + def time_in_drawdown(self): - dd=self.drawdown() - dd=[z[0] for z in dd.values if not np.isnan(z[0])] - in_dd=float(len([z for z in dd if z<0])) - return in_dd/float(len(dd)) + dd = self.drawdown() + dd = [z[0] for z in dd.values if not np.isnan(z[0])] + in_dd = float(len([z for z in dd if z < 0])) + return in_dd / float(len(dd)) def calmar(self): - return self.ann_daily_mean()/-self.worst_drawdown() + return self.ann_daily_mean() / -self.worst_drawdown() def avg_return_to_drawdown(self): - return self.ann_daily_mean()/-self.avg_drawdown() + return self.ann_daily_mean() / -self.avg_drawdown() def sortino(self): - daily_stddev=np.std(self.losses()) - daily_mean=self.mean() - - ann_stdev=daily_stddev*ROOT_BDAYS_INYEAR - ann_mean=daily_mean*BUSINESS_DAYS_IN_YEAR - - return ann_mean/ann_stdev - + daily_stddev = np.std(self.losses()) + daily_mean = self.mean() + + ann_stdev = daily_stddev * ROOT_BDAYS_INYEAR + ann_mean = daily_mean * BUSINESS_DAYS_IN_YEAR + + return ann_mean / ann_stdev + def vals(self): - x=self.daily() - x=[z[0] for z in x.values if not np.isnan(z[0])] + x = self.daily() + x = [z[0] for z in x.values if not np.isnan(z[0])] return x def min(self): - + return np.nanmin(self.vals()) - + def max(self): return np.max(self.vals()) - + def median(self): return np.median(self.vals()) - + def skew(self): return skew(self.vals()) - + def mean(self): - x=self.daily() + x = self.daily() return np.nanmean(x) def std(self): - x=self.daily() + x = self.daily() return np.nanstd(x) - + def losses(self): - x=self.vals() - return [z for z in x if z<0] - + x = self.vals() + return [z for z in x if z < 0] + def gains(self): - x=self.vals() - return [z for z in x if z>0] - + x = self.vals() + return [z for z in x if z > 0] + def avg_loss(self): return np.mean(self.losses()) def avg_gain(self): return np.mean(self.gains()) - - + def gaintolossratio(self): - return self.avg_gain()/-self.avg_loss() - + return self.avg_gain() / -self.avg_loss() + def profitfactor(self): - return sum(self.gains())/-sum(self.losses()) - + return sum(self.gains()) / -sum(self.losses()) + def hitrate(self): - no_gains=float(len(self.gains())) - no_losses=float(len(self.losses())) - return no_gains/(no_losses+no_gains) - + no_gains = float(len(self.gains())) + no_losses = float(len(self.losses())) + return no_gains / (no_losses + no_gains) + def rolling_ann_std(self, window=40): - x=self.daily() - y=pd.rolling_std(x, window, min_periods=4, center=True) - return y*ROOT_BDAYS_INYEAR - + x = self.daily() + y = pd.rolling_std(x, window, min_periods=4, center=True) + return y * ROOT_BDAYS_INYEAR def stats(self): - - stats_list=["min", "max", "median", "mean", "std", "skew", - "ann_daily_mean", "ann_daily_std", "sharpe", "sortino", - "avg_drawdown", "time_in_drawdown", - "calmar", "avg_return_to_drawdown", + + stats_list = ["min", "max", "median", "mean", "std", "skew", + "ann_daily_mean", "ann_daily_std", "sharpe", "sortino", + "avg_drawdown", "time_in_drawdown", + "calmar", "avg_return_to_drawdown", "avg_loss", "avg_gain", "gaintolossratio", "profitfactor", "hitrate"] - - build_stats=[] + + build_stats = [] for stat_name in stats_list: - stat_method=getattr(self, stat_name) - ans=stat_method() + stat_method = getattr(self, stat_name) + ans = stat_method() build_stats.append((stat_name, "{0:.4g}".format(ans))) - comment1=("You can also plot:", ["rolling_ann_std", "drawdown", "curve"]) - comment2=("You can also print:", ["weekly", "monthly", "annual"]) - + comment1 = ("You can also plot:", [ + "rolling_ann_std", "drawdown", "curve"]) + comment2 = ("You can also print:", ["weekly", "monthly", "annual"]) + return [build_stats, comment1, comment2] - + if __name__ == '__main__': import doctest doctest.testmod() diff --git a/syscore/algos.py b/syscore/algos.py index e05ab6c519..95548bde44 100644 --- a/syscore/algos.py +++ b/syscore/algos.py @@ -6,22 +6,23 @@ """ import pandas as pd -def robust_vol_calc(x, days=35, min_periods=10, vol_abs_min=0.0000000001, vol_floor=True, - floor_min_quant=0.05, floor_min_periods=100, + +def robust_vol_calc(x, days=35, min_periods=10, vol_abs_min=0.0000000001, vol_floor=True, + floor_min_quant=0.05, floor_min_periods=100, floor_days=500): """ Robust exponential volatility calculation, assuming daily series of prices We apply an absolute minimum level of vol (absmin); and a volfloor based on lowest vol over recent history - + :param days: Number of days in lookback (*default* 35) :type days: int - :param min_periods: The minimum number of observations (*default* 10) + :param min_periods: The minimum number of observations (*default* 10) :type min_periods: int - + :param vol_abs_min: The size of absolute minimum (*default* =0.0000000001) 0.0= not used - :type absmin: float or None - + :type absmin: float or None + :param vol_floor Apply a floor to volatility (*default* True) :type vol_floor: bool :param floor_min_quant: The quantile to use for volatility floor (eg 0.05 means we use 5% vol) (*default 0.05) @@ -30,32 +31,31 @@ def robust_vol_calc(x, days=35, min_periods=10, vol_abs_min=0.0000000001, vol_fl :type floor_days: int :param floor_min_periods: Minimum observations for floor - until reached floor is zero (*default* 100) :type floor_min_periods: int - + :returns: pd.DataFrame -- volatility measure - - + + """ - ## Standard deviation will be nan for first 10 non nan values - vol=pd.ewmstd(x, span=days,min_periods=min_periods) - - vol[vol>> expiry_date('201503') datetime.datetime(2015, 3, 1, 0, 0) >>> expiry_date('20150305') datetime.datetime(2015, 3, 5, 0, 0) - + >>> expiry_date(datetime.datetime(2015,5,1)) datetime.datetime(2015, 5, 1, 0, 0) - + """ - - if type(expiry_ident)==str: - ## do string expiry calc - if len(expiry_ident)==6: - expiry_date=datetime.datetime.strptime(expiry_ident, "%Y%m") - elif len(expiry_ident)==8: - expiry_date=datetime.datetime.strptime(expiry_ident, "%Y%m%d") + if isinstance(expiry_ident, str): + # do string expiry calc + if len(expiry_ident) == 6: + expiry_date = datetime.datetime.strptime(expiry_ident, "%Y%m") + elif len(expiry_ident) == 8: + expiry_date = datetime.datetime.strptime(expiry_ident, "%Y%m%d") else: raise Exception("") - - elif type(expiry_ident)==datetime.datetime or type(expiry_ident)==datetime.date: - expiry_date=expiry_ident - + + elif isinstance(expiry_ident, datetime.datetime) or isinstance(expiry_ident, datetime.date): + expiry_date = expiry_ident + else: - raise Exception("expiry_date needs to be a string with 4 or 6 digits, ") - - ## 'Natural' form is datetime + raise Exception( + "expiry_date needs to be a string with 4 or 6 digits, ") + + # 'Natural' form is datetime return expiry_date def expiry_diff(carry_row, floor_date_diff=20): """ Given a pandas row containing CARRY_CONTRACT and PRICE_CONTRACT, both of which represent dates - + Return the annualised difference between the dates - + :param carry_row: object with attributes CARRY_CONTRACT and PRICE_CONTRACT :type carry_row: pandas row, or something that quacks like it - + :param floor_date_diff: If date resolves to less than this, floor here (*default* 20) :type carry_row: pandas row, or something that quacks like it - + :returns: datetime.datetime or datetime.date - + """ - if carry_row.PRICE_CONTRACT=="" or carry_row.CARRY_CONTRACT=="": + if carry_row.PRICE_CONTRACT == "" or carry_row.CARRY_CONTRACT == "": return np.nan - ans=float((expiry_date(carry_row.CARRY_CONTRACT) - expiry_date(carry_row.PRICE_CONTRACT)).days) - if abs(ans)>> + >>> """ - path_as_list=name_with_dots.rsplit(".") - - ## join last two things together. This is probably the most obfuscated code I've ever written - if len(path_as_list)>=2: - path_as_list[-1]=path_as_list[-2]+"."+path_as_list.pop() + path_as_list = name_with_dots.rsplit(".") - return get_pathname_for_package_from_list(path_as_list) + # join last two things together. This is probably the most obfuscated code + # I've ever written + if len(path_as_list) >= 2: + path_as_list[-1] = path_as_list[-2] + "." + path_as_list.pop() + return get_pathname_for_package_from_list(path_as_list) def get_pathname_for_package(name_with_dots): """ Returns the pathname of part of a package - + :param name_with_dots: Path and file name written with "." eg "sysdata.tests" :type str: - + :returns: full pathname of package eg "../sysdata/tests/" - - + + """ - path_as_list=name_with_dots.rsplit(".") - + path_as_list = name_with_dots.rsplit(".") return get_pathname_for_package_from_list(path_as_list) - def get_pathname_for_package_from_list(path_as_list): """ Returns the filename of part of a package - + :param path_as_list: List of path and file name eg ["syscore","fileutils.py"] :type path_as_list: - + :returns: full pathname of package """ - package_name=path_as_list[0] - paths_or_files=path_as_list[1:] + package_name = path_as_list[0] + paths_or_files = path_as_list[1:] d = os.path.dirname(sys.modules[package_name].__file__) - if len(paths_or_files)==0: + if len(paths_or_files) == 0: return d - - last_item_in_list=path_as_list.pop() - pathname = os.path.join(get_pathname_for_package_from_list(path_as_list), last_item_in_list) + + last_item_in_list = path_as_list.pop() + pathname = os.path.join(get_pathname_for_package_from_list( + path_as_list), last_item_in_list) return pathname if __name__ == '__main__': import doctest - doctest.testmod() \ No newline at end of file + doctest.testmod() diff --git a/syscore/genutils.py b/syscore/genutils.py index 53e751e245..17583397d0 100644 --- a/syscore/genutils.py +++ b/syscore/genutils.py @@ -5,19 +5,18 @@ from math import copysign - def str_of_int(x): """ Returns the string of int of x, handling nan's or whatever - + :param x: Name of python package :type x: int or float - + :returns: 1.0 or -1.0 - + >>> str_of_int(34) '34' - + >>> str_of_int(34.0) '34' @@ -26,39 +25,38 @@ def str_of_int(x): '' """ - if type(x) is int: + if isinstance(x, int): return str(x) try: return str(int(x)) except: return "" - - + def sign(x): """ Return the sign of x (float or int) :param x: Thing we want sign of :type x: int, float - + :returns: 1 or -1 - + >>> sign(3) 1.0 - + >>> sign(3.0) 1.0 - + >>> sign(-3) -1.0 - + >>> sign(0) 1.0 - - + + """ return copysign(1, x) if __name__ == '__main__': import doctest - doctest.testmod() \ No newline at end of file + doctest.testmod() diff --git a/syscore/objects.py b/syscore/objects.py index 2c9d1310e0..814ede3fbb 100644 --- a/syscore/objects.py +++ b/syscore/objects.py @@ -8,53 +8,56 @@ def resolve_function(func_or_func_name): """ if func_or_func_name is a callable function, then return the function - + If it is a string containing '.' then it is a function in a module, return the function :param func_or_func_name: Name of a function including package.module, or a callable function :type func_or_func_name: int, float, or something else eg np.nan - + :returns: function - + >>> resolve_function(str) - + >>> resolve_function("syscore.algos.robust_vol_calc") - + """ if hasattr(func_or_func_name, "__call__"): - ## it's a function, just return it so can be used - ## doesn't work for all functions, but we only care about callable ones + # it's a function, just return it so can be used + # doesn't work for all functions, but we only care about callable ones return func_or_func_name - - if not type(func_or_func_name)==str: - raise Exception("Called resolve_function with non string or callable object %s" % str(func_or_func_name)) + + if not isinstance(func_or_func_name, str): + raise Exception("Called resolve_function with non string or callable object %s" % str( + func_or_func_name)) if "." in func_or_func_name: - ## it's another module, have to get it - mod_name, func_name = func_or_func_name.rsplit('.',1) + # it's another module, have to get it + mod_name, func_name = func_or_func_name.rsplit('.', 1) mod = importlib.import_module(mod_name) func = getattr(mod, func_name, None) else: - raise Exception("Need full module file name string: %s isn't good enough" % func_or_func_name) - + raise Exception( + "Need full module file name string: %s isn't good enough" % func_or_func_name) + return func + def resolve_data_method(some_object, data_string): """ eg if data_string="data1.data2.method" then returns the method some_object.data1.data2.method :param some_object: The object with a method :type some_object: object - + :param data_string: method or attribute within object :type data_string: str - + :returns: method in some_object - + >>> from sysdata.data import Data >>> >>> data=Data() @@ -65,21 +68,21 @@ def resolve_data_method(some_object, data_string): >>> setattr(meta_data, "meta", data) >>> resolve_data_method(meta_data, "meta.get_instrument_price") - >>> + >>> >>> meta_meta_data=Data() >>> setattr(meta_meta_data, "moremeta", meta_data) >>> resolve_data_method(meta_meta_data, "moremeta.meta.get_instrument_price") - + """ - list_to_parse=data_string.rsplit(".") - + list_to_parse = data_string.rsplit(".") + def _get_attr_within_list(an_object, list_to_parse): - if len(list_to_parse)==0: + if len(list_to_parse) == 0: return an_object - next_attr=list_to_parse.pop(0) - sub_object=getattr(an_object, next_attr) + next_attr = list_to_parse.pop(0) + sub_object = getattr(an_object, next_attr) return _get_attr_within_list(sub_object, list_to_parse) return _get_attr_within_list(some_object, list_to_parse) @@ -87,10 +90,10 @@ def _get_attr_within_list(an_object, list_to_parse): def update_recalc(stage_object, additional_protected=[]): """ - Update the recalculation dictionary - + Update the recalculation dictionary + Used when a stage inherits from another - + (see system.stage and systems.futures.rawdata for an example) :param stage_object: The stage object with attributes to test @@ -98,15 +101,15 @@ def update_recalc(stage_object, additional_protected=[]): :param additional_protected: Things to add to this attribute :type additional_protected: list of str - + :returns: None (changes stage_object) """ - - original_dont_delete=getattr(stage_object,"_protected", []) - - child_dont_delete=list(set(original_dont_delete+additional_protected)) - + + original_dont_delete = getattr(stage_object, "_protected", []) + + child_dont_delete = list(set(original_dont_delete + additional_protected)) + setattr(stage_object, "_protected", child_dont_delete) @@ -119,7 +122,7 @@ def hasallattr(some_object, attrlist=[]): :param attrlist: Attributes to check :type attrlist: list of str - + :returns: bool >>> from sysdata.data import Data @@ -131,11 +134,11 @@ def hasallattr(some_object, attrlist=[]): >>> hasallattr(data, ["one", "two","three"]) False - - + + """ return all([hasattr(some_object, attrname) for attrname in attrlist]) if __name__ == '__main__': import doctest - doctest.testmod() \ No newline at end of file + doctest.testmod() diff --git a/syscore/pdutils.py b/syscore/pdutils.py index e63f706f66..18f15fe138 100644 --- a/syscore/pdutils.py +++ b/syscore/pdutils.py @@ -6,23 +6,25 @@ import numpy as np from syscore.fileutils import get_filename_for_package + def pd_readcsv_frompackage(filename): """ - Run pd_readcsv on a file in python + Run pd_readcsv on a file in python :param args: List showing location in project directory of file eg systems, provided, tests.csv :type args: str - + :returns: pd.DataFrame - + """ - - full_filename=get_filename_for_package(filename) + + full_filename = get_filename_for_package(filename) return pd_readcsv(full_filename) + def pd_readcsv(filename, date_index_name="DATETIME"): - """ - Reads a pandas data frame, with time index labelled + """ + Reads a pandas data frame, with time index labelled package_name(/path1/path2.., filename :param filename: Filename with extension @@ -30,21 +32,22 @@ def pd_readcsv(filename, date_index_name="DATETIME"): :param date_index_name: Column name of date index :type date_index_name: list of str - - + + :returns: pd.DataFrame - + """ - - ans=pd.read_csv(filename) - ans.index=pd.to_datetime(ans[date_index_name]).values - + + ans = pd.read_csv(filename) + ans.index = pd.to_datetime(ans[date_index_name]).values + del ans[date_index_name] - - ans.index.name=None + + ans.index.name = None return ans + def apply_cap(pd_dataframe, capvalue): """ Applies a cap to the values in a Tx1 pandas dataframe @@ -54,10 +57,10 @@ def apply_cap(pd_dataframe, capvalue): :param capvalue: Maximum absolute value allowed :type capvlue: int or float - - + + :returns: pd.DataFrame Tx1 - + >>> x=pd.DataFrame(dict(a=[2.0, 7.0, -7.0, -6.99]), pd.date_range(pd.datetime(2015,1,1), periods=4)) >>> apply_cap(x, 5.0) a @@ -65,25 +68,26 @@ def apply_cap(pd_dataframe, capvalue): 2015-01-02 5 2015-01-03 -5 2015-01-04 -5 - + """ pd.date_range - ## Will do weird things otherwise - assert capvalue>0 - - ## create max and min columns - max_ts=pd.Series([capvalue]*pd_dataframe.shape[0], pd_dataframe.index) - min_ts=pd.Series([-capvalue]*pd_dataframe.shape[0], pd_dataframe.index) - - joined_ts=pd.concat([pd_dataframe, max_ts], axis=1) - joined_ts=joined_ts.min(axis=1) - joined_ts=pd.concat([joined_ts, min_ts], axis=1) - joined_ts=joined_ts.max(axis=1).to_frame(pd_dataframe.columns[0]) - - joined_ts[np.isnan(pd_dataframe)]=np.nan + # Will do weird things otherwise + assert capvalue > 0 + + # create max and min columns + max_ts = pd.Series([capvalue] * pd_dataframe.shape[0], pd_dataframe.index) + min_ts = pd.Series([-capvalue] * pd_dataframe.shape[0], pd_dataframe.index) + + joined_ts = pd.concat([pd_dataframe, max_ts], axis=1) + joined_ts = joined_ts.min(axis=1) + joined_ts = pd.concat([joined_ts, min_ts], axis=1) + joined_ts = joined_ts.max(axis=1).to_frame(pd_dataframe.columns[0]) + + joined_ts[np.isnan(pd_dataframe)] = np.nan return joined_ts -def index_match(x,y,ffill): + +def index_match(x, y, ffill): """ Join together two pd.DataFrames into a 2xT @@ -95,38 +99,37 @@ def index_match(x,y,ffill): :param y: Tx1 pandas data frame :type y: pd.DataFrame - + :param ffill: should we ffill x and y respectively :type ffill: 2-tuple (bool, bool) - + :returns: pd.DataFrame Tx2 """ - (ffill_x, ffill_y)=ffill - - ans=pd.concat([x,y], axis=1, join='inner') + (ffill_x, ffill_y) = ffill + + ans = pd.concat([x, y], axis=1, join='inner') - if ffill_x or ffill_y: - - jointts=ans.index + if ffill_x or ffill_y: + + jointts = ans.index if ffill_x: - xnew=x.ffill().reindex(jointts) + xnew = x.ffill().reindex(jointts) else: - xnew=x.reindex(jointts) - + xnew = x.reindex(jointts) if ffill_y: - ynew=y.ffill().reindex(jointts) + ynew = y.ffill().reindex(jointts) else: - ynew=y.reindex(jointts) - - ans=pd.concat([xnew,ynew], axis=1) - + ynew = y.reindex(jointts) + + ans = pd.concat([xnew, ynew], axis=1) + return ans -def divide_df_single_column(x,y, ffill=(False, False)): +def divide_df_single_column(x, y, ffill=(False, False)): """ Divide Tx1 dataframe by Tx1 dataframe @@ -138,12 +141,12 @@ def divide_df_single_column(x,y, ffill=(False, False)): :param y: Tx1 pandas data frame :type y: pd.DataFrame - + :param ffill: should we ffill x and y respectively :type ffill: 2-tuple (bool, bool) - + :returns: pd.DataFrame Tx1 - + >>> x=pd.DataFrame(dict(a=[2.0, 7.0, -7.0, -7.00]), pd.date_range(pd.datetime(2015,1,1), periods=4)) >>> y=pd.DataFrame(dict(b=[2.0, 3.5, 2.0, -3.5]), pd.date_range(pd.datetime(2015,1,1), periods=4)) >>> divide_df_single_column(x,y) @@ -152,17 +155,18 @@ def divide_df_single_column(x,y, ffill=(False, False)): 2015-01-02 2.0 2015-01-03 -3.5 2015-01-04 2.0 - - + + """ - ans=index_match(x,y,ffill) - - ans=ans.iloc[:,0]/ans.iloc[:,1] - ans=ans.to_frame(x.columns[0]) - + ans = index_match(x, y, ffill) + + ans = ans.iloc[:, 0] / ans.iloc[:, 1] + ans = ans.to_frame(x.columns[0]) + return ans -def multiply_df_single_column(x,y, ffill=(False, False)): + +def multiply_df_single_column(x, y, ffill=(False, False)): """ Multiply Tx1 dataframe by Tx1 dataframe; time indicies don't have to match @@ -171,9 +175,9 @@ def multiply_df_single_column(x,y, ffill=(False, False)): :param y: Tx1 pandas data frame :type y: pd.DataFrame - + :returns: pd.DataFrame Tx1 - + >>> x=pd.DataFrame(dict(a=range(10)), pd.date_range(pd.datetime(2015,1,1), periods=10)) >>> y=pd.DataFrame(dict(b=range(10)), pd.date_range(pd.datetime(2015,1,5), periods=10)) >>> multiply_df_single_column(x,y) @@ -184,34 +188,34 @@ def multiply_df_single_column(x,y, ffill=(False, False)): 2015-01-08 21 2015-01-09 32 2015-01-10 45 - + """ - - ans=index_match(x,y,ffill) - - ans=ans.iloc[:,0]*ans.iloc[:,1] - ans=ans.to_frame(x.columns[0]) - + + ans = index_match(x, y, ffill) + + ans = ans.iloc[:, 0] * ans.iloc[:, 1] + ans = ans.to_frame(x.columns[0]) + return ans - -def multiply_df(x,y): + +def multiply_df(x, y): """ - Multiply TxN dataframe by TxN dataframe + Multiply TxN dataframe by TxN dataframe :param x: Tx1 pandas data frame :type x: pd.DataFrame :param y: Tx1 pandas data frame :type y: pd.DataFrame - + :returns: pd.DataFrame Tx1 - + >>> x=pd.DataFrame(dict(a=[2.0, 7.0, -7.0, -7.00]), pd.date_range(pd.datetime(2015,1,1), periods=4)) >>> y=pd.DataFrame(dict(b=[2.0, 3.0, 2.0, -3.0]), pd.date_range(pd.datetime(2015,1,1), periods=4)) >>> multiply_df(x,y) a - 2015-01-01 4 + 2015-01-01 4 2015-01-02 21 2015-01-03 -14 2015-01-04 21 @@ -222,75 +226,73 @@ def multiply_df(x,y): a b 2015-01-01 -4 21 2015-01-02 14 21 - + """ - assert x.shape==y.shape - ans=pd.concat([x.iloc[:,cidx]*y.iloc[:,cidx] for cidx in range(x.shape[1])], axis=1) - ans.columns=x.columns - - return ans + assert x.shape == y.shape + ans = pd.concat([x.iloc[:, cidx] * y.iloc[:, cidx] + for cidx in range(x.shape[1])], axis=1) + ans.columns = x.columns + return ans def fix_weights_vs_pdm(weights, pdm): """ Take a matrix of weights and positions/forecasts (pdm) - - Ensure that the weights in each row add up to 1, for active positions/forecasts (not np.nan values after forward filling) - + + Ensure that the weights in each row add up to 1, for active positions/forecasts (not np.nan values after forward filling) + This deals with the problem of different rules and/or instruments having different history - - :param weights: Weights to + + :param weights: Weights to :type weights: TxK pd.DataFrame (same columns as weights, perhaps different length) - + :param pdm: :type pdm: TxK pd.DataFrame (same columns as weights, perhaps different length) - + :returns: TxK pd.DataFrame of adjusted weights """ - - - ## forward fill forecasts/positions - pdm_ffill=pdm.ffill() - - ## resample weights - adj_weights=weights.reindex(pdm_ffill.index, method='ffill') - - ## ensure columns are aligned - adj_weights=adj_weights[pdm.columns] - - ## remove weights if nan forecast - adj_weights[np.isnan(pdm_ffill)]=0.0 - - ## change rows so weights add to one + + # forward fill forecasts/positions + pdm_ffill = pdm.ffill() + + # resample weights + adj_weights = weights.reindex(pdm_ffill.index, method='ffill') + + # ensure columns are aligned + adj_weights = adj_weights[pdm.columns] + + # remove weights if nan forecast + adj_weights[np.isnan(pdm_ffill)] = 0.0 + + # change rows so weights add to one def _sum_row_fix(weight_row): - swr=sum(weight_row) - if swr==0.0: + swr = sum(weight_row) + if swr == 0.0: return weight_row - new_weights=weight_row/swr + new_weights = weight_row / swr return new_weights - - adj_weights=adj_weights.apply(_sum_row_fix,1 ) + + adj_weights = adj_weights.apply(_sum_row_fix, 1) return adj_weights + def drawdown(x): - """ Returns a ts of drawdowns for a time series x - + :param x: account curve (cumulated returns) :param x: pd.DataFrame or Series - + :returns: pd.DataFrame or Series """ - maxx=pd.rolling_max(x, 99999999, min_periods=1) + maxx = pd.rolling_max(x, 99999999, min_periods=1) return x - maxx - if __name__ == '__main__': import doctest - doctest.testmod() \ No newline at end of file + doctest.testmod() diff --git a/syscore/tests/test_Accounting.py b/syscore/tests/test_Accounting.py index f844437c40..c9ff4b1f4c 100644 --- a/syscore/tests/test_Accounting.py +++ b/syscore/tests/test_Accounting.py @@ -8,58 +8,74 @@ import pandas as pd import numpy as np -from syscore.accounting import pandl, get_positions_from_forecasts, get_trades_from_positions +from syscore.accounting import pandl, get_positions_from_forecasts, get_trades_from_positions -class Test(unittest.TestCase): +class Test(unittest.TestCase): def test_get_positions_from_forecasts(self): - fx=pd.DataFrame([2.0]*10 , pd.date_range(start=pd.datetime(2014,12,30), periods=10)) - price= pd.DataFrame([100, 103, 105, 106, 110, 105, np.nan, 106, 120, np.nan, 142], pd.date_range(start=pd.datetime(2015,1,1), periods=11)) - forecast=pd.DataFrame([np.nan, np.nan, np.nan, np.nan,10.0, 10.0, 15.0, 15.0, 5.0, 0.0, -5.0], pd.date_range(start=pd.datetime(2015,1,1), periods=11)) - value_of_price_point=150.0 - position=get_positions_from_forecasts(price, None, forecast, fx, value_of_price_point, min_periods=1) - - self.assertAlmostEqual(list(position.position.values)[4:], [2523.4937866824253, 905.72296272461699, 1358.5844440869255, 1358.5844440869255, 248.78044993282995, 0.0, -248.78044993282995]) + fx = pd.DataFrame( + [2.0] * 10, pd.date_range(start=pd.datetime(2014, 12, 30), periods=10)) + price = pd.DataFrame([100, 103, 105, 106, 110, 105, np.nan, 106, + 120, np.nan, 142], pd.date_range(start=pd.datetime(2015, 1, 1), periods=11)) + forecast = pd.DataFrame([np.nan, np.nan, np.nan, np.nan, 10.0, 10.0, 15.0, 15.0, + 5.0, 0.0, -5.0], pd.date_range(start=pd.datetime(2015, 1, 1), periods=11)) + value_of_price_point = 150.0 + position = get_positions_from_forecasts( + price, None, forecast, fx, value_of_price_point, min_periods=1) + + self.assertAlmostEqual(list(position.position.values)[4:], [ + 2523.4937866824253, 905.72296272461699, 1358.5844440869255, 1358.5844440869255, 248.78044993282995, 0.0, -248.78044993282995]) def test_get_trades_from_positions(self): - positions= pd.DataFrame([np.nan, 2, 3, np.nan, 2, 3, 3.1, 4, 3, 5, 7], pd.date_range(start=pd.datetime(2015,1,1), periods=11)) - price= pd.DataFrame([100, 103, np.nan, 106, 110, 105, np.nan, 106, 120, np.nan, 142], pd.date_range(start=pd.datetime(2015,1,1), periods=11)) + positions = pd.DataFrame([np.nan, 2, 3, np.nan, 2, 3, 3.1, 4, + 3, 5, 7], pd.date_range(start=pd.datetime(2015, 1, 1), periods=11)) + price = pd.DataFrame([100, 103, np.nan, 106, 110, 105, np.nan, 106, + 120, np.nan, 142], pd.date_range(start=pd.datetime(2015, 1, 1), periods=11)) #trades=get_trades_from_positions(price, positions, delayfill, roundpositions, None, None, None, None) - trades=get_trades_from_positions(price, positions, True, True, None, None, None, None) - - self.assertEqual(list(trades.trades), [2.0, 1.0, -1.0, 1.0, 1.0, -1.0, 2.0, 2.0]) - self.assertEqual(list(trades.fill_price)[:-1], [106.0, 106.0, 105.0, 106.0, 120.0, 142.0, 142.0]) + trades = get_trades_from_positions( + price, positions, True, True, None, None, None, None) + self.assertEqual(list(trades.trades), [ + 2.0, 1.0, -1.0, 1.0, 1.0, -1.0, 2.0, 2.0]) + self.assertEqual(list(trades.fill_price)[ + :-1], [106.0, 106.0, 105.0, 106.0, 120.0, 142.0, 142.0]) - trades=get_trades_from_positions(price, positions, False, True, None, None, None, None) + trades = get_trades_from_positions( + price, positions, False, True, None, None, None, None) - self.assertEqual(list(trades.trades), [2.0, 1.0, -1.0, 1.0, 1.0, -1.0, 2.0, 2.0]) - self.assertEqual(list(trades.fill_price), [103.0, 106.0, 110.0, 105.0, 106.0, 120.0, 142.0, 142.0]) + self.assertEqual(list(trades.trades), [ + 2.0, 1.0, -1.0, 1.0, 1.0, -1.0, 2.0, 2.0]) + self.assertEqual(list(trades.fill_price), [ + 103.0, 106.0, 110.0, 105.0, 106.0, 120.0, 142.0, 142.0]) + trades = get_trades_from_positions( + price, positions, True, False, None, None, None, None) - trades=get_trades_from_positions(price, positions, True, False, None, None, None, None) - - self.assertEqual(list(trades.trades), [2.0, 1.0, -1.0, 1.0, 0.1, 0.9, -1.0, 2.0, 2.0]) - self.assertEqual(list(trades.fill_price)[:-1], [106.0, 106.0, 105.0, 106.0, 106.0, 120.0, 120.0, 142.0, 142.0]) + self.assertEqual(list(trades.trades), [ + 2.0, 1.0, -1.0, 1.0, 0.1, 0.9, -1.0, 2.0, 2.0]) + self.assertEqual(list(trades.fill_price)[ + :-1], [106.0, 106.0, 105.0, 106.0, 106.0, 120.0, 120.0, 142.0, 142.0]) def test_pandl(self): - fx=pd.DataFrame([2.0]*10 , pd.date_range(start=pd.datetime(2014,12,30), periods=10)) - price= pd.DataFrame( [100, 103, 105, 106, 110, 105, 104.5, - np.nan, 120, np.nan, 142], pd.date_range(start=pd.datetime(2015,1,1), periods=11)) - trades=pd.concat([pd.DataFrame(dict(trades=[ 2, 1, -1, np.nan, 1 ], - fill_price=[ 102.9, 105.5, 106.5, np.nan, 106.]), - pd.date_range(start=pd.datetime(2015,1,2), periods=5)), - pd.DataFrame(dict(trades=[ -1, 1, -1], - fill_price= [ 107, 119, 132 ]), pd.date_range(start=pd.datetime(2015,1,8), periods=3))]) - - - ans=pandl(price, trades, marktomarket=True, fx=fx) - self.assertEqual(list(ans.pandl_base)[1:], [0.0, 10.4, 6.,14., -16., -9., 15., 48., 78., 40.]) - - ans2=pandl(price, trades, marktomarket=False, fx=fx) - self.assertEqual(list(ans2.pandl_base)[1:], [ 10.4, 6., 0., -2., 6., 48., 78.]) - + fx = pd.DataFrame( + [2.0] * 10, pd.date_range(start=pd.datetime(2014, 12, 30), periods=10)) + price = pd.DataFrame([100, 103, 105, 106, 110, 105, 104.5, + np.nan, 120, np.nan, 142], pd.date_range(start=pd.datetime(2015, 1, 1), periods=11)) + trades = pd.concat([pd.DataFrame(dict(trades=[2, 1, -1, np.nan, 1], + fill_price=[102.9, 105.5, 106.5, np.nan, 106.]), + pd.date_range(start=pd.datetime(2015, 1, 2), periods=5)), + pd.DataFrame(dict(trades=[-1, 1, -1], + fill_price=[107, 119, 132]), pd.date_range(start=pd.datetime(2015, 1, 8), periods=3))]) + + ans = pandl(price, trades, marktomarket=True, fx=fx) + self.assertEqual(list(ans.pandl_base)[1:], [ + 0.0, 10.4, 6., 14., -16., -9., 15., 48., 78., 40.]) + + ans2 = pandl(price, trades, marktomarket=False, fx=fx) + self.assertEqual(list(ans2.pandl_base)[1:], [ + 10.4, 6., 0., -2., 6., 48., 78.]) + if __name__ == "__main__": #import sys;sys.argv = ['', 'Test.testName'] - unittest.main() \ No newline at end of file + unittest.main() diff --git a/syscore/tests/test_algos.py b/syscore/tests/test_algos.py index d40ed8c64d..c10817b12d 100644 --- a/syscore/tests/test_algos.py +++ b/syscore/tests/test_algos.py @@ -12,45 +12,46 @@ class Test(ut.TestCase): - def test_robust_vol_calc(self): - prices=pd_readcsv_frompackage("syscore", "pricetestdata.csv", ["tests"]) - returns=prices.diff() - vol=robust_vol_calc(returns, days=35) - self.assertAlmostEqual(vol.iloc[-1,0], 0.41905275480464305) + prices = pd_readcsv_frompackage( + "syscore", "pricetestdata.csv", ["tests"]) + returns = prices.diff() + vol = robust_vol_calc(returns, days=35) + self.assertAlmostEqual(vol.iloc[-1, 0], 0.41905275480464305) + + vol = robust_vol_calc(returns, days=100) + self.assertAlmostEqual(vol.iloc[-1, 0], 0.43906619578902956) - vol=robust_vol_calc(returns, days=100) - self.assertAlmostEqual(vol.iloc[-1,0], 0.43906619578902956) - - def test_robust_vol_calc_min_period(self): - prices=pd_readcsv_frompackage("syscore", "pricetestdata_min_period.csv", ["tests"]) - returns=prices.diff() - vol=robust_vol_calc(returns, min_periods=9) - self.assertAlmostEqual(vol.iloc[-1,0], 0.45829858614978286) - vol=robust_vol_calc(returns, min_periods=10) + prices = pd_readcsv_frompackage( + "syscore", "pricetestdata_min_period.csv", ["tests"]) + returns = prices.diff() + vol = robust_vol_calc(returns, min_periods=9) + self.assertAlmostEqual(vol.iloc[-1, 0], 0.45829858614978286) + vol = robust_vol_calc(returns, min_periods=10) self.assertTrue(np.isnan(vol.iloc[-1][0])) - - + def test_robust_vol_calc_min_value(self): - prices=pd_readcsv_frompackage("syscore", "pricetestdata_zero_vol.csv", ["tests"]) - returns=prices.diff() - vol=robust_vol_calc(returns, vol_abs_min=0.01) - self.assertEqual(vol.iloc[-1,0], 0.01) + prices = pd_readcsv_frompackage( + "syscore", "pricetestdata_zero_vol.csv", ["tests"]) + returns = prices.diff() + vol = robust_vol_calc(returns, vol_abs_min=0.01) + self.assertEqual(vol.iloc[-1, 0], 0.01) def test_robust_vol_calc_floor(self): - prices=pd_readcsv_frompackage("syscore", "pricetestdata_vol_floor.csv", ["tests"]) - returns=prices.diff() - vol=robust_vol_calc(returns) - self.assertAlmostEqual(vol.iloc[-1,0], 0.54492982003602064) - vol=robust_vol_calc(returns, vol_floor=False) - self.assertAlmostEqual(vol.iloc[-1,0], 0.42134038479240132) - vol=robust_vol_calc(returns, floor_min_quant=.5) - self.assertAlmostEqual(vol.iloc[-1,0], 1.6582199589924964) - vol=robust_vol_calc(returns, floor_min_periods=500) - self.assertAlmostEqual(vol.iloc[-1,0], 0.42134038479240132) - vol=robust_vol_calc(returns, floor_days=10, floor_min_periods=5) - self.assertAlmostEqual(vol.iloc[-1,0], 0.42134038479240132) + prices = pd_readcsv_frompackage( + "syscore", "pricetestdata_vol_floor.csv", ["tests"]) + returns = prices.diff() + vol = robust_vol_calc(returns) + self.assertAlmostEqual(vol.iloc[-1, 0], 0.54492982003602064) + vol = robust_vol_calc(returns, vol_floor=False) + self.assertAlmostEqual(vol.iloc[-1, 0], 0.42134038479240132) + vol = robust_vol_calc(returns, floor_min_quant=.5) + self.assertAlmostEqual(vol.iloc[-1, 0], 1.6582199589924964) + vol = robust_vol_calc(returns, floor_min_periods=500) + self.assertAlmostEqual(vol.iloc[-1, 0], 0.42134038479240132) + vol = robust_vol_calc(returns, floor_days=10, floor_min_periods=5) + self.assertAlmostEqual(vol.iloc[-1, 0], 0.42134038479240132) """ def test_calc_ewmac_forecast(self): @@ -64,4 +65,4 @@ def test_calc_ewmac_forecast(self): if __name__ == "__main__": #import sys;sys.argv = ['', 'Test.test_robust_vol_calc'] - ut.main() \ No newline at end of file + ut.main() diff --git a/syscore/tests/test_dateutils.py b/syscore/tests/test_dateutils.py index 5d60970ad7..c45e83ef19 100644 --- a/syscore/tests/test_dateutils.py +++ b/syscore/tests/test_dateutils.py @@ -13,15 +13,16 @@ class Test(ut.TestCase): def test_data(self): - x=pd.DataFrame(dict(CARRY_CONTRACT=["", "201501", "", "201501", "20150101", "20150101","201501"], - PRICE_CONTRACT=["", "", "201504", "201504", "201504", "20150115","201406"])) - + x = pd.DataFrame(dict(CARRY_CONTRACT=["", "201501", "", "201501", "20150101", "20150101", "201501"], + PRICE_CONTRACT=["", "", "201504", "201504", "201504", "20150115", "201406"])) + return x def test_expiry_diff(self): - x=self.test_data() - expiries=x.apply(expiry_diff,1) - expected=[-0.24640657084188911, -0.24640657084188911, -0.054757015742642023, 0.58590006844626963] + x = self.test_data() + expiries = x.apply(expiry_diff, 1) + expected = [-0.24640657084188911, -0.24640657084188911, - + 0.054757015742642023, 0.58590006844626963] self.assertTrue(all([np.isnan(y) for y in expiries[:3]])) for (got, wanted) in zip(expiries[3:], expected): self.assertAlmostEqual(got, wanted) @@ -29,4 +30,3 @@ def test_expiry_diff(self): if __name__ == "__main__": #import sys;sys.argv = ['', 'Test.test_robust_vol_calc'] ut.main() - diff --git a/syscore/tests/test_pdutils.py b/syscore/tests/test_pdutils.py index 26b56ac7df..2119fed4ab 100644 --- a/syscore/tests/test_pdutils.py +++ b/syscore/tests/test_pdutils.py @@ -8,20 +8,23 @@ import numpy as np from syscore.pdutils import divide_df_single_column, multiply_df, multiply_df_single_column -class Test(unittest.TestCase): +class Test(unittest.TestCase): def test_divide_df_single_column(self): - x=pd.DataFrame(dict(a=[2.0, 7.0, -7.0, -7.00, 3.5]), pd.date_range(pd.datetime(2015,1,1), periods=5)) - y=pd.DataFrame(dict(b=[2.0, 3.5, 2.0, -3.5, -3.5]), pd.date_range(pd.datetime(2015,1,1), periods=5)) - ans=list(divide_df_single_column(x,y).iloc[:,0]) + x = pd.DataFrame(dict(a=[2.0, 7.0, -7.0, -7.00, 3.5]), + pd.date_range(pd.datetime(2015, 1, 1), periods=5)) + y = pd.DataFrame(dict(b=[2.0, 3.5, 2.0, -3.5, -3.5]), + pd.date_range(pd.datetime(2015, 1, 1), periods=5)) + ans = list(divide_df_single_column(x, y).iloc[:, 0]) self.assertEqual(ans, [1., 2., -3.5, 2., -1.]) - x=pd.DataFrame(dict(a=[2.0, np.nan, -7.0, np.nan, 3.5]), pd.date_range(pd.datetime(2015,1,1), periods=5)) - y=pd.DataFrame(dict(b= [2.0, 3.5, np.nan, np.nan, -3.5]), pd.date_range(pd.datetime(2015,1,2), periods=5)) - - - ans=list(divide_df_single_column(x, y).iloc[:,0]) + x = pd.DataFrame(dict(a=[2.0, np.nan, -7.0, np.nan, 3.5]), + pd.date_range(pd.datetime(2015, 1, 1), periods=5)) + y = pd.DataFrame(dict(b=[2.0, 3.5, np.nan, np.nan, -3.5]), + pd.date_range(pd.datetime(2015, 1, 2), periods=5)) + + ans = list(divide_df_single_column(x, y).iloc[:, 0]) self.assertTrue(np.isnan(ans[0])) self.assertTrue(np.isnan(ans[2])) @@ -29,22 +32,25 @@ def test_divide_df_single_column(self): self.assertEqual(ans[1], -2.0) - ans=list(divide_df_single_column(x, y, ffill=(True, False)).iloc[:,0]) + ans = list(divide_df_single_column( + x, y, ffill=(True, False)).iloc[:, 0]) self.assertEqual(ans[0], 1.0) - - ans=list(divide_df_single_column(x, y, ffill=(False, True)).iloc[:,0]) + + ans = list(divide_df_single_column( + x, y, ffill=(False, True)).iloc[:, 0]) self.assertEqual(ans[3], 1.0) - - ans=list(divide_df_single_column(x, y, ffill=(True, True)).iloc[:,0]) + + ans = list(divide_df_single_column( + x, y, ffill=(True, True)).iloc[:, 0]) self.assertEqual(list(ans), [1., -2., -2.0, 1.]) def multiply_df_single_column(self): pass - + def multiply_df(self): pass if __name__ == "__main__": #import sys;sys.argv = ['', 'Test.testName'] - unittest.main() \ No newline at end of file + unittest.main() diff --git a/sysdata/configdata.py b/sysdata/configdata.py index 724f15fd08..3462882197 100644 --- a/sysdata/configdata.py +++ b/sysdata/configdata.py @@ -12,81 +12,81 @@ import yaml from syscore.fileutils import get_filename_for_package + class Config(object): - - def __init__(self, config_object=dict()): + + def __init__(self, config_object=dict()): """ - Config objects control the behaviour of systems - + Config objects control the behaviour of systems + :param config_object: Eithier: - a string (which points to a YAML filename) + a string (which points to a YAML filename) or a dict (which may nest many things) or a list of strings or dicts (build config from multiple elements, latter elements will overwrite earlier oness) - + :type config_object: str or dict - + :returns: new Config object - + >>> Config(dict(parameters=dict(p1=3, p2=4.6), another_thing=[])) Config with elements: another_thing, parameters >>> Config("sysdata.tests.exampleconfig.yaml") Config with elements: parameters, trading_rules - + >>> Config(["sysdata.tests.exampleconfig.yaml", dict(parameters=dict(p1=3, p2=4.6), another_thing=[])]) Config with elements: another_thing, parameters, trading_rules """ - if type(config_object) is list: - ## multiple configs + if isinstance(config_object, list): + # multiple configs for config_item in config_object: self._create_config_from_item(config_item) else: self._create_config_from_item(config_object) def _create_config_from_item(self, config_item): - if type(config_item) is dict: - ## its a dict + if isinstance(config_item, dict): + # its a dict self._create_config_from_dict(config_item) - - elif type(config_item) is str: - ## must be a file YAML'able, from which we load the - filename=get_filename_for_package(config_item) + + elif isinstance(config_item, str): + # must be a file YAML'able, from which we load the + filename = get_filename_for_package(config_item) with open(filename) as file_to_parse: - dict_to_parse=yaml.load(file_to_parse) - + dict_to_parse = yaml.load(file_to_parse) + self._create_config_from_dict(dict_to_parse) - - + else: - raise Exception("Can only create a config with a nested dict or the string of a 'yamable' filename, or a list comprising these things") - + raise Exception( + "Can only create a config with a nested dict or the string of a 'yamable' filename, or a list comprising these things") def _create_config_from_dict(self, config_object): """ - Take a dictionary object and turn it into self - + Take a dictionary object and turn it into self + When we've finished self will be an object where the attributes are - + So if config_objec=dict(a=2, b=2) Then this object will become self.a=2, self.b=2 - + """ - attr_names=list(config_object.keys()) - [setattr(self, keyname, config_object[keyname]) for keyname in config_object] - existing_elements=getattr(self, "_elements", []) - new_elements=list(set(existing_elements+attr_names)) - + attr_names = list(config_object.keys()) + [setattr(self, keyname, config_object[keyname]) + for keyname in config_object] + existing_elements = getattr(self, "_elements", []) + new_elements = list(set(existing_elements + attr_names)) + setattr(self, "_elements", new_elements) - + def __repr__(self): - element_names=getattr(self, "_elements", []) - element_names.sort() - element_names=", ".join(element_names) - return "Config with elements: "+element_names - - + element_names = sorted(getattr(self, "_elements", [])) + element_names = ", ".join(element_names) + return "Config with elements: " + element_names + + if __name__ == '__main__': import doctest - doctest.testmod() \ No newline at end of file + doctest.testmod() diff --git a/sysdata/csvdata.py b/sysdata/csvdata.py index dfd4e325ac..58fc06cf5d 100644 --- a/sysdata/csvdata.py +++ b/sysdata/csvdata.py @@ -17,53 +17,53 @@ """ Static variables to store location of data """ -LEGACY_DATA_PATH="sysdata.legacycsv" +LEGACY_DATA_PATH = "sysdata.legacycsv" + class csvFuturesData(FuturesData): """ Get futures specific data from legacy csv files - - Extends the FuturesData class for a specific data source - + + Extends the FuturesData class for a specific data source + """ - + def __init__(self, datapath=None): """ Create a FuturesData object for reading .csv files from datapath inherits from FuturesData We look for data in .csv files - - :param datapath: path to find .csv files (defaults to LEGACY_DATA_MODULE/LEGACY_DATA_DIR + + :param datapath: path to find .csv files (defaults to LEGACY_DATA_MODULE/LEGACY_DATA_DIR :type datapath: None or str - + :returns: new csvFuturesData object - + >>> data=csvFuturesData() >>> data FuturesData object with 38 instruments - + """ - if datapath is None: - datapath=LEGACY_DATA_PATH - - datapath=get_pathname_for_package(datapath) + if datapath is None: + datapath = LEGACY_DATA_PATH + + datapath = get_pathname_for_package(datapath) """ Most Data objects that read data from a specific place have a 'source' of some kind Here it's a directory """ setattr(self, "_datapath", datapath) - - + def get_instrument_price(self, instrument_code): """ Get instrument price - - :param instrument_code: instrument to get prices for + + :param instrument_code: instrument to get prices for :type instrument_code: str - + :returns: pd.DataFrame >>> data=csvFuturesData("sysdata.tests") @@ -76,25 +76,25 @@ def get_instrument_price(self, instrument_code): 2015-04-21 129.390625 2015-04-22 128.867188 """ - - ## Read from .csv - filename=os.path.join(self._datapath, instrument_code+"_price.csv") - instrpricedata=pd_readcsv(filename) - instrpricedata.columns=["price"] - + + # Read from .csv + filename = os.path.join(self._datapath, instrument_code + "_price.csv") + instrpricedata = pd_readcsv(filename) + instrpricedata.columns = ["price"] + return instrpricedata - + def get_instrument_raw_carry_data(self, instrument_code): """ Returns a pd. dataframe with the 4 columns PRICE, CARRY, PRICE_CONTRACT, CARRY_CONTRACT - + These are specifically needed for futures trading - - :param instrument_code: instrument to get carry data for + + :param instrument_code: instrument to get carry data for :type instrument_code: str - + :returns: pd.DataFrame - + >>> data=csvFuturesData("sysdata.tests") >>> data.get_instrument_raw_carry_data("EDOLLAR").tail(5) PRICE CARRY CARRY_CONTRACT PRICE_CONTRACT @@ -105,39 +105,43 @@ def get_instrument_raw_carry_data(self, instrument_code): 2015-04-22 NaN 97.8325 201806 201809 """ - filename=os.path.join(self._datapath, instrument_code+"_carrydata.csv") - instrcarrydata=pd_readcsv(filename) - instrcarrydata.columns=["PRICE", "CARRY", "CARRY_CONTRACT", "PRICE_CONTRACT"] + filename = os.path.join( + self._datapath, instrument_code + "_carrydata.csv") + instrcarrydata = pd_readcsv(filename) + instrcarrydata.columns = ["PRICE", "CARRY", + "CARRY_CONTRACT", "PRICE_CONTRACT"] + + instrcarrydata.CARRY_CONTRACT = instrcarrydata.CARRY_CONTRACT.apply( + str_of_int) + instrcarrydata.PRICE_CONTRACT = instrcarrydata.PRICE_CONTRACT.apply( + str_of_int) - instrcarrydata.CARRY_CONTRACT=instrcarrydata.CARRY_CONTRACT.apply(str_of_int) - instrcarrydata.PRICE_CONTRACT=instrcarrydata.PRICE_CONTRACT.apply(str_of_int) - return instrcarrydata def _get_instrument_data(self): """ Get a data frame of interesting information about instruments, eithier from a file or cached - + :returns: pd.DataFrame >>> data=csvFuturesData("sysdata.tests") >>> data._get_instrument_data() Instrument Pointsize AssetClass Currency - Instrument + Instrument EDOLLAR EDOLLAR 2500 STIR USD US10 US10 1000 Bond USD """ - - filename=os.path.join(self._datapath, "instrumentconfig.csv") - instr_data=pd.read_csv(filename) - instr_data.index=instr_data.Instrument + + filename = os.path.join(self._datapath, "instrumentconfig.csv") + instr_data = pd.read_csv(filename) + instr_data.index = instr_data.Instrument return instr_data def get_instrument_list(self): """ list of instruments in this data set - + :returns: list of str >>> data=csvFuturesData("sysdata.tests") @@ -147,11 +151,9 @@ def get_instrument_list(self): ['EDOLLAR', 'US10'] """ - instr_data=self._get_instrument_data() - - return list(instr_data.Instrument) + instr_data = self._get_instrument_data() - + return list(instr_data.Instrument) def get_instrument_asset_classes(self): """ @@ -163,19 +165,19 @@ def get_instrument_asset_classes(self): EDOLLAR STIR US10 Bond Name: AssetClass, dtype: object - """ - instr_data=self._get_instrument_data() - instr_assets=instr_data.AssetClass - + """ + instr_data = self._get_instrument_data() + instr_assets = instr_data.AssetClass + return instr_assets def get_value_of_block_price_move(self, instrument_code): """ How much is a $1 move worth in value terms? - - :param instrument_code: instrument to get value for + + :param instrument_code: instrument to get value for :type instrument_code: str - + :returns: float >>> data=csvFuturesData("sysdata.tests") @@ -183,18 +185,18 @@ def get_value_of_block_price_move(self, instrument_code): 2500 """ - instr_data=self._get_instrument_data() - block_move_value=instr_data.loc[instrument_code,'Pointsize'] - + instr_data = self._get_instrument_data() + block_move_value = instr_data.loc[instrument_code, 'Pointsize'] + return block_move_value - + def get_instrument_currency(self, instrument_code): """ What is the currency that this instrument is priced in? - - :param instrument_code: instrument to get value for + + :param instrument_code: instrument to get value for :type instrument_code: str - + :returns: str >>> data=csvFuturesData("sysdata.tests") @@ -202,21 +204,21 @@ def get_instrument_currency(self, instrument_code): 'USD' """ - instr_data=self._get_instrument_data() - currency=instr_data.loc[instrument_code,'Currency'] - + instr_data = self._get_instrument_data() + currency = instr_data.loc[instrument_code, 'Currency'] + return currency - + def _get_fx_data(self, currency1, currency2): """ Get fx data - - :param currency1: numerator currency + + :param currency1: numerator currency :type currency1: str - :param currency2: denominator currency + :param currency2: denominator currency :type currency2: str - + :returns: Tx1 pd.DataFrame, or None if not available >>> data=csvFuturesData() @@ -229,22 +231,22 @@ def _get_fx_data(self, currency1, currency2): fx 2015-10-30 0.718219 2015-11-02 0.714471 - """ + """ - if currency1==currency2: + if currency1 == currency2: return self._get_default_series() - - filename=os.path.join(self._datapath, "%s%sfx.csv" % (currency1, currency2)) + filename = os.path.join( + self._datapath, "%s%sfx.csv" % (currency1, currency2)) try: - fxdata=pd_readcsv(filename) + fxdata = pd_readcsv(filename) except: return None - - fxdata.columns=["%s%s" % (currency1, currency2)] - + + fxdata.columns = ["%s%s" % (currency1, currency2)] + return fxdata - + if __name__ == '__main__': import doctest doctest.testmod() diff --git a/sysdata/data.py b/sysdata/data.py index 45439a84c6..593d316f50 100644 --- a/sysdata/data.py +++ b/sysdata/data.py @@ -1,120 +1,117 @@ import pandas as pd - - class Data(object): - + """ Core data object - Base class - + Data objects are used to get data from a particular source, and give certain information about it - + The bare Data class isn't much good and holds only price data - - Normally we'd inherit from this for specific asset classes (eg carry data for futures), and then for a + + Normally we'd inherit from this for specific asset classes (eg carry data for futures), and then for a specific source of data (eg csv files, databases, ...) - + The inheritance is: - + Base generic class: Data -> asset class specific eg futuresdata.FuturesData -> source specific eg legacy.csvFuturesData - + """ - - + def __init__(self): - """ - Data socket base class + Data socket base class """ def __repr__(self): - return "Data object with %d instruments" % len(self.get_instrument_list()) - + return "Data object with %d instruments" % len( + self.get_instrument_list()) def get_instrument_price(self, instrument_code): """ Default method to get instrument price Will usually be overriden when inherited with specific data source - - :param instrument_code: instrument to get prices for + + :param instrument_code: instrument to get prices for :type instrument_code: str - + :returns: pd.DataFrame - + """ - raise Exception("You have created a Data() object; you might need to use a more specific data object" % instrument_code) + raise Exception( + "You have created a Data() object; you might need to use a more specific data object" % instrument_code) - def __getitem__(self, keyname): """ convenience method to get the price, make it look like a dict - - :param keyname: instrument to get prices for + + :param keyname: instrument to get prices for :type keyname: str - - :returns: pd.DataFrame + + :returns: pd.DataFrame """ - price=self.get_instrument_price(keyname) - - return price + price = self.get_instrument_price(keyname) + return price def get_instrument_list(self): """ list of instruments in this data set - + :returns: list of str - + """ return [] def keys(self): """ list of instruments in this data set - + :returns: list of str - + >>> data=Data() >>> data.keys() [] """ return self.get_instrument_list() - + def get_value_of_block_price_move(self, instrument_code): """ How much does a $1 (or whatever) move in the price of an instrument block affect it's value? eg 100.0 for 100 shares - - :param instrument_code: instrument to value for + + :param instrument_code: instrument to value for :type instrument_code: str - - :returns: float - + + :returns: float + """ - + return 1.0 def _get_default_currency(self): """ We assume we always have rates for the default currency vs others to use in getting cross rates - eg if default is USD assume we always know GBPUSD, AUDUSD... - + eg if default is USD assume we always know GBPUSD, AUDUSD... + :returns: str - + """ - DEFAULT_CURRENCY="USD" - + DEFAULT_CURRENCY = "USD" + return DEFAULT_CURRENCY def _get_default_series(self): """ What we return if currency rates match """ - DEFAULT_DATES=pd.date_range(start=pd.datetime(1970,1,1),end=pd.datetime(2050,1,1)) - DEFAULT_RATE_SERIES=pd.DataFrame(dict(fx=[1.0]*len(DEFAULT_DATES)), index=DEFAULT_DATES) + DEFAULT_DATES = pd.date_range(start=pd.datetime( + 1970, 1, 1), end=pd.datetime(2050, 1, 1)) + DEFAULT_RATE_SERIES = pd.DataFrame( + dict(fx=[1.0] * len(DEFAULT_DATES)), index=DEFAULT_DATES) return DEFAULT_RATE_SERIES @@ -124,82 +121,84 @@ def get_instrument_currency(self, instrument_code): Since we don't have any actual data unless we overload this object, just return the default - :param instrument_code: instrument to value for + :param instrument_code: instrument to value for :type instrument_code: str :returns: str - + """ return self._get_default_currency() - def _get_fx_data(self, currency1, currency2): """ Get the FX rate currency1/currency2 between two currencies Or return None if not available - + (Normally we'd over ride this with a specific source) - :param instrument_code: instrument to value for + :param instrument_code: instrument to value for :type instrument_code: str - :param base_currency: instrument to value for + :param base_currency: instrument to value for :type instrument_code: str - + :returns: Tx1 pd.DataFrame, or None if not found - - + + """ - if currency1==currency2: + if currency1 == currency2: return self._get_default_series() - ## no data available + # no data available return None def _get_fx_cross(self, currency1, currency2): """ Get the FX rate between two currencies, using crosses with DEFAULT_CURRENCY if neccessary - :param instrument_code: instrument to value for + :param instrument_code: instrument to value for :type instrument_code: str - :param base_currency: instrument to value for + :param base_currency: instrument to value for :type instrument_code: str - + :returns: Tx1 pd.DataFrame - - + + """ - - ### try and get from raw data - fx_rate_series=self._get_fx_data(currency1, currency2) - + + # try and get from raw data + fx_rate_series = self._get_fx_data(currency1, currency2) + if fx_rate_series is None: - ## missing; have to get get cross rates - default_currency=self._get_default_currency() - currency1_vs_default=self._get_fx_data(currency1, default_currency).resample("1B", how="last") - currency2_vs_default=self._get_fx_data(currency2, default_currency).resample("1B", how="last") - - together=pd.concat([currency1_vs_default, currency2_vs_default], axis=1, join='inner').ffill() - - fx_rate_series=together.iloc[:,0]/together.iloc[:, 1] - fx_rate_series=fx_rate_series.to_frame("fx") - + # missing; have to get get cross rates + default_currency = self._get_default_currency() + currency1_vs_default = self._get_fx_data( + currency1, default_currency).resample("1B", how="last") + currency2_vs_default = self._get_fx_data( + currency2, default_currency).resample("1B", how="last") + + together = pd.concat( + [currency1_vs_default, currency2_vs_default], axis=1, join='inner').ffill() + + fx_rate_series = together.iloc[:, 0] / together.iloc[:, 1] + fx_rate_series = fx_rate_series.to_frame("fx") + return fx_rate_series def get_fx_for_instrument(self, instrument_code, base_currency): """ Get the FX rate between the FX rate for the instrument and the base (account) currency - :param instrument_code: instrument to value for + :param instrument_code: instrument to value for :type instrument_code: str - :param base_currency: instrument to value for + :param base_currency: instrument to value for :type instrument_code: str - + :returns: Tx1 pd.DataFrame - + >>> data=Data() >>> data.get_fx_for_instrument("wibble", "USD").tail(2) fx @@ -207,12 +206,12 @@ def get_fx_for_instrument(self, instrument_code, base_currency): 2050-01-01 1 """ - instrument_currency=self.get_instrument_currency(instrument_code) - fx_rate_series=self._get_fx_cross(instrument_currency, base_currency) - + instrument_currency = self.get_instrument_currency(instrument_code) + fx_rate_series = self._get_fx_cross(instrument_currency, base_currency) + return fx_rate_series if __name__ == '__main__': import doctest - doctest.testmod() \ No newline at end of file + doctest.testmod() diff --git a/sysdata/futuresdata.py b/sysdata/futuresdata.py index a8fe1f1f05..02b5fff88d 100644 --- a/sysdata/futuresdata.py +++ b/sysdata/futuresdata.py @@ -4,37 +4,38 @@ class FuturesData(Data): """ Get futures specific data - - Extends the Data class to add additional features for asset specific - + + Extends the Data class to add additional features for asset specific + Will normally be overriden by a method for a specific data source See legacy.py - + """ def __repr__(self): - return "FuturesData object with %d instruments" % len(self.get_instrument_list()) - + return "FuturesData object with %d instruments" % len( + self.get_instrument_list()) def get_instrument_raw_carry_data(self, instrument_code): """ Returns a pd. dataframe with the 4 columns PRICE, CARRY, PRICE_CONTRACT, CARRY_CONTRACT - + These are specifically needed for futures trading - + For other asset classes we'd probably pop in eg equities fundamental data, FX interest rates... - + Normally we'd inherit from this method for a specific data source - - :param instrument_code: instrument to get carry data for + + :param instrument_code: instrument to get carry data for :type instrument_code: str - + :returns: pd.DataFrame - + """ - ### Default method to get instrument price - ### Will usually be overriden when inherited with specific data source - raise Exception("You have created a FuturesData() object or you probably need to replace this method to do anything useful") + # Default method to get instrument price + # Will usually be overriden when inherited with specific data source + raise Exception( + "You have created a FuturesData() object or you probably need to replace this method to do anything useful") if __name__ == '__main__': diff --git a/sysdata/quandl.py b/sysdata/quandl.py index be54ca920e..90cbcdb136 100644 --- a/sysdata/quandl.py +++ b/sysdata/quandl.py @@ -2,4 +2,3 @@ Get data from quandl NOT YET IMPLEMENTED """ - diff --git a/systems/account.py b/systems/account.py index 04897601ff..2f905d2114 100644 --- a/systems/account.py +++ b/systems/account.py @@ -3,70 +3,70 @@ from syscore.accounting import accountCurve, pandl from systems.stage import SystemStage from systems.basesystem import ALL_KEYNAME -from syscore.pdutils import multiply_df_single_column, fix_weights_vs_pdm +from syscore.pdutils import multiply_df_single_column, fix_weights_vs_pdm + class Account(SystemStage): """ SystemStage for accounting - - KEY INPUT: + + KEY INPUT: found in self.get_forecast(instrument_code, rule_variation) found in self.get_forecast_weights_and_fdm(instrument_code) system.positionSize.get_subsystem_position(instrument_code) found in self.get_subsystem_position(instrument_code) - + found in self.get_portfolio_position() - + found in self.get_instrument_weights_and_idm(instrument_code) found in self.get_capital() - + KEY OUTPUT: self.forecasts() (will be used to optimise forecast weights in future version) - + self.instruments() (will be used to optimise instrument weights in future version) - + Name: accounts """ - + def __init__(self): """ Create a SystemStage for accounting - - - + + + """ - delete_on_recalc=[] + delete_on_recalc = [] + + dont_delete = [] - dont_delete=[] - setattr(self, "_delete_on_recalc", delete_on_recalc) setattr(self, "_dont_recalc", dont_delete) setattr(self, "name", "accounts") - def get_subsystem_position(self, instrument_code): """ Get the position assuming all capital in one instruments, from a previous module - + :param instrument_code: instrument to get values for :type instrument_code: str - - :returns: Tx1 pd.DataFrame - + + :returns: Tx1 pd.DataFrame + KEY INPUT - + >>> from systems.tests.testdata import get_test_object_futures_with_pos_sizing >>> from systems.basesystem import System >>> (posobject, combobject, capobject, rules, rawdata, data, config)=get_test_object_futures_with_pos_sizing() >>> system=System([rawdata, rules, posobject, combobject, capobject,PortfoliosFixed()], data, config) - >>> + >>> >>> ## from config >>> system.portfolio.get_subsystem_position("EDOLLAR").tail(2) ss_position @@ -76,30 +76,35 @@ def get_subsystem_position(self, instrument_code): """ return self.parent.positionSize.get_subsystem_position(instrument_code) - + def portfolio(self): pass - - def instrument(self, subset=None, percentage=True, isolated=False, sumup=False): + + def instrument(self, subset=None, percentage=True, + isolated=False, sumup=False): pass - - def rules(self, subset=None, percentage=True, isolated=False, sumup=False): + + def rules(self, subset=None, percentage=True, + isolated=False, sumup=False): pass - def rulegroup(self, subset=None, percentage=True, isolated=False, sumup=False): + def rulegroup(self, subset=None, percentage=True, + isolated=False, sumup=False): pass - - def rulestyle(self, subset=None, percentage=True, isolated=False, sumup=False): + + def rulestyle(self, subset=None, percentage=True, + isolated=False, sumup=False): pass - ## these should be in a futures accounting object... - def assetclass(self, subset=None, percentage=True, isolated=False, sumup=False): + # these should be in a futures accounting object... + def assetclass(self, subset=None, percentage=True, + isolated=False, sumup=False): pass - - def country(self, subset=None, percentage=True, isolated=False, sumup=False): + + def country(self, subset=None, percentage=True, + isolated=False, sumup=False): pass - - + if __name__ == '__main__': import doctest diff --git a/systems/basesystem.py b/systems/basesystem.py index 22d600e5a3..dea8d36c91 100644 --- a/systems/basesystem.py +++ b/systems/basesystem.py @@ -2,113 +2,113 @@ """ This is used for items which affect an entire system, not just one instrument """ -ALL_KEYNAME="all" +ALL_KEYNAME = "all" class System(object): ''' system objects are used for signal processing in a 'tree' like framework - + This is the base class which all systems inherit - + Systems are: - + made up of stages - + take a data, and optionally a config object - - - ''' - def __init__(self, stage_list, data, config=None): + ''' + + def __init__(self, stage_list, data, config=None): """ Create a system object for doing simulations or live trading - :param data: data for doing simulations + :param data: data for doing simulations :type data: sysdata.data.Data (or anything that inherits from that) - - :param config: Optional configuration - :type config: sysdata.configdata.Config - :param stage_list: A list of stages + :param config: Optional configuration + :type config: sysdata.configdata.Config + + :param stage_list: A list of stages :type stage_list: list of systems.stage.SystemStage (or anything that inherits from it) - + :returns: new system object - + >>> from stage import SystemStage >>> stage=SystemStage() >>> from sysdata.csvdata import csvFuturesData >>> data=csvFuturesData() >>> System([stage], data) System with stages: unnamed - + """ - + if config is None: - ## Default - for very dull systems this is sufficient - config=Config() - + # Default - for very dull systems this is sufficient + config = Config() + setattr(self, "data", data) setattr(self, "config", config) - protected=[] - stage_names=[] - - assert type(stage_list) is list - + protected = [] + stage_names = [] + + assert isinstance(stage_list, list) + for stage in stage_list: - + """ This is where we put the methods to store various stages of the process - + """ - ## Each stage has a link back to the parent system + # Each stage has a link back to the parent system setattr(stage, "parent", self) - - ## Stages have names, which are also how we find them in the system attributes - sub_name=stage.name - + + # Stages have names, which are also how we find them in the system + # attributes + sub_name = stage.name + if sub_name in stage_names: - raise Exception("You have duplicate subsystems with the name %s. Remove one of them, or change a name." % sub_name) + raise Exception( + "You have duplicate subsystems with the name %s. Remove one of them, or change a name." % sub_name) setattr(self, sub_name, stage) stage_names.append(sub_name) - - stage_protected=getattr(stage, "_protected",[]) - protected=protected+stage_protected - + + stage_protected = getattr(stage, "_protected", []) + protected = protected + stage_protected + setattr(self, "_stage_names", stage_names) - + """ The cache hides all intermediate results - + We call optimal_positions and then that propogates back finding all the data we need - - The results are then cached in the object. Should we call delete_instrument_data (in base class system) then + + The results are then cached in the object. Should we call delete_instrument_data (in base class system) then everything related to a particular instrument is removed from these 'nodes' This is very useful in live trading when we don't want to update eg cross sectional data every sample """ - + setattr(self, "_cache", dict()) setattr(self, "_protected", protected) - - + def __repr__(self): - sslist=", ".join(self._stage_names) - return "System with stages: "+sslist - + sslist = ", ".join(self._stage_names) + return "System with stages: " + sslist + """ A cache lives inside each system object, storing preliminary results There are 3 kinds of things in a cache with different levels of persistence: - anything that isn't special - - things that have an 'all' key - + - things that have an 'all' key - - _protected - that wouldn't normally be deleted - + """ def get_items_with_data(self): @@ -116,37 +116,38 @@ def get_items_with_data(self): Return items in the cache with data (or at least key values set) :returns: list of str """ - + return list(self._cache.keys()) - - + def get_protected_items(self): return self._protected - + def get_instrument_codes_for_item(self, itemname): if itemname in self._cache: return self._cache[itemname].keys() else: return [] - + def get_items_for_instrument(self, instrument_code): """ Return all key items which have instrument_code keys - + :returns: list of str """ - - items_with_data=self.get_items_with_data() - items_code_list=[self.get_instrument_codes_for_item(itemname) for itemname in items_with_data] - items_with_instrument_data=[itemname for (itemname, code_list) in zip(items_with_data, items_code_list) if instrument_code in code_list] - + + items_with_data = self.get_items_with_data() + items_code_list = [self.get_instrument_codes_for_item( + itemname) for itemname in items_with_data] + items_with_instrument_data = [itemname for (itemname, code_list) in zip( + items_with_data, items_code_list) if instrument_code in code_list] + return items_with_instrument_data def delete_item(self, itemname): """ Delete everything in cache related to itemname - :param itemname: Instrument to delete + :param itemname: Instrument to delete :type itemname: str Will kill protected things as well @@ -154,251 +155,251 @@ def delete_item(self, itemname): if itemname not in self._cache: return None - + return self._cache.pop(itemname) - def delete_items_for_instrument(self, instrument_code, delete_protected=False): + def delete_items_for_instrument( + self, instrument_code, delete_protected=False): """ Delete everything in the system relating to an instrument_code - :param instrument_code: Instrument to delete + :param instrument_code: Instrument to delete :type instrument_code: str - :param deleted_protected: Delete everything, even stuff in self.protected? + :param deleted_protected: Delete everything, even stuff in self.protected? :type delete_protected: bool - + [When working with a live system we listen to a message bus - + if a new price is received then we delete things in the cache - + This means when we ask for self.optimal_positions(instrument_code) it has to recalc all - intermediate steps as the cached + intermediate steps as the cached However we ignore anything in self._protected This is normally cross sectional data which we only want to calculate periodically - + if delete_protected is True then we delete that stuff as well (this is roughly equivalent to creating the systems object from scratch) - + """ - item_list=self.get_items_for_instrument(instrument_code) + item_list = self.get_items_for_instrument(instrument_code) if not delete_protected: - protected_items=self.get_protected_items() - item_list=[itemname for itemname in item_list if itemname not in protected_items] - - deleted_values=[self._delete_item_from_cache(itemname, instrument_code) for itemname in item_list] - + protected_items = self.get_protected_items() + item_list = [ + itemname for itemname in item_list if itemname not in protected_items] + + deleted_values = [self._delete_item_from_cache( + itemname, instrument_code) for itemname in item_list] + return deleted_values def get_items_across_system(self): return self.get_items_for_instrument(ALL_KEYNAME) - + def delete_items_across_system(self, delete_protected=False): - return self.delete_items_for_instrument(ALL_KEYNAME, delete_protected=delete_protected) - + return self.delete_items_for_instrument( + ALL_KEYNAME, delete_protected=delete_protected) + def delete_all_items(self, delete_protected=False): - item_list=self.get_items_with_data() + item_list = self.get_items_with_data() if not delete_protected: - protected_items=self.get_protected_items() - item_list=[itemname for itemname in item_list if itemname not in protected_items] - - deleted_values=[self.delete_item(itemname) for itemname in item_list] - + protected_items = self.get_protected_items() + item_list = [ + itemname for itemname in item_list if itemname not in protected_items] + + deleted_values = [self.delete_item(itemname) for itemname in item_list] + return deleted_values - - def get_item_from_cache(self, itemname, instrument_code=ALL_KEYNAME, keyname=None): + + def get_item_from_cache( + self, itemname, instrument_code=ALL_KEYNAME, keyname=None): """ Get an item from the cache self._cache - - :param itemname: The item to get + + :param itemname: The item to get :type itemname: str - + :param instrument_code: The instrument to get it from :type instrument_code: str - + :param keyname: The further key (eg rule variation name) to get for a nested item :type keyname: str - + :returns: None or item """ - + if itemname not in self._cache: - ## no cache for this item yet + # no cache for this item yet return None - + if instrument_code not in self._cache[itemname]: return None - + if keyname is None: - ## one level dict, and we know we have an answer + # one level dict, and we know we have an answer return self._cache[itemname][instrument_code] else: if keyname not in self._cache[itemname][instrument_code]: - ## missing in nested dict + # missing in nested dict return None - - ## nested dict and we have an answer + + # nested dict and we have an answer return self._cache[itemname][instrument_code][keyname] - - ## should never get here, failsafe + + # should never get here, failsafe return None - def _delete_item_from_cache(self, itemname, instrument_code=ALL_KEYNAME, keyname=None): - + def _delete_item_from_cache( + self, itemname, instrument_code=ALL_KEYNAME, keyname=None): """ Delete an item from the cache self._cache - + Returns the deleted value, or None if not available - - :param itemname: The item to get + + :param itemname: The item to get :type itemname: str - + :param instrument_code: The instrument to get it from :type instrument_code: str - + :param keyname: The further key (eg rule variation name) to get for a nested item :type keyname: str - + :returns: None or item """ - + if itemname not in self._cache: return None - + if instrument_code not in self._cache[itemname]: return None - + if keyname is None: - ## one level dict, and we know we have an answer + # one level dict, and we know we have an answer return self._cache[itemname].pop(instrument_code) else: if keyname not in self._cache[itemname][instrument_code]: - ## missing in nested dict + # missing in nested dict return None - - ## nested dict and we have an answer + + # nested dict and we have an answer return self._cache[itemname][instrument_code].pop(keyname) - - ## should never get here + + # should never get here return None - def set_item_in_cache(self, value, itemname, instrument_code=ALL_KEYNAME, keyname=None): + def set_item_in_cache(self, value, itemname, + instrument_code=ALL_KEYNAME, keyname=None): """ Set an item in a cache to a specific value - + If any part of the cache 'tree' is missing then adds it - + :param value: The value to set to :type value: Anything (normally pd.frames or floats) - + :param itemname: The item to set :type itemname: str - + :param instrument_code: The instrument to set :type instrument_code: str - + :param keyname: The further key (eg rule variation name) to set for a nested item :type keyname: str - + :returns: None or item """ - + if itemname not in self._cache: - ## no cache for this item yet, let's set one up - self._cache[itemname]=dict() - + # no cache for this item yet, let's set one up + self._cache[itemname] = dict() + if keyname is None: - ## one level dict - self._cache[itemname][instrument_code]=value + # one level dict + self._cache[itemname][instrument_code] = value else: - ## nested + # nested if instrument_code not in self._cache[itemname]: - ## missing dict let's add it - self._cache[itemname][instrument_code]=dict() - - self._cache[itemname][instrument_code][keyname]=value - - return value - + # missing dict let's add it + self._cache[itemname][instrument_code] = dict() + self._cache[itemname][instrument_code][keyname] = value + return value def calc_or_cache(self, itemname, instrument_code, func, *args, **kwargs): """ Assumes that self._cache has an attribute itemname, and that is a dict - + If self._cache.itemname[instrument_code] exists return it. Else call func with *args and **kwargs if the latter updates the dictionary - - :param itemname: attribute of object containing a dict + + :param itemname: attribute of object containing a dict :type itemname: str - - :param instrument_code: keyname to look for in dict + + :param instrument_code: keyname to look for in dict :type instrument_code: str - + :param func: function to call if missing from cache. will take self and instrument_code as first two args :type func: function - + :param args, kwargs: also passed to func if called - + :returns: contents of dict or result of calling function - - + + """ - value=self.get_item_from_cache(itemname, instrument_code) - + value = self.get_item_from_cache(itemname, instrument_code) + if value is None: - - value=func(self, instrument_code, *args, **kwargs) + + value = func(self, instrument_code, *args, **kwargs) self.set_item_in_cache(value, itemname, instrument_code) - + return value - - - - - def calc_or_cache_nested(self, itemname, instrument_code, keyname, func, *args, **kwargs): + + def calc_or_cache_nested( + self, itemname, instrument_code, keyname, func, *args, **kwargs): """ Assumes that self._cache has a key itemname, and that is a nested dict - - If itemname[instrument_code][keyname] exists return it. + + If itemname[instrument_code][keyname] exists return it. Else call func with arguments: self, instrument_code, keyname, *args and **kwargs if we have to call the func updates the dictionary with it's value - + Used for cache within various kinds of objects like config, price, data, system... - - :param itemname: cache item to look for + + :param itemname: cache item to look for :type itemname: str - - :param instrument_code: keyname to look for in dict + + :param instrument_code: keyname to look for in dict :type instrument_code: str - - :param keyname: keyname to look for in nested dict + + :param keyname: keyname to look for in nested dict :type keyname: valid dict key - + :param func: function to call if missing from cache. will take self and instrument_code, keyname as first three args :type func: function - + :param args, kwargs: also passed to func if called - + :returns: contents of dict or result of calling function - - + + """ - - value=self.get_item_from_cache(itemname, instrument_code, keyname) - - if value is None: - value=func(self, instrument_code, keyname, *args, **kwargs) + + value = self.get_item_from_cache(itemname, instrument_code, keyname) + + if value is None: + value = func(self, instrument_code, keyname, *args, **kwargs) self.set_item_in_cache(value, itemname, instrument_code, keyname) - + return value - - - - + + if __name__ == '__main__': import doctest doctest.testmod() diff --git a/systems/defaults.py b/systems/defaults.py index 1e7a75b1d5..b0b5f9cc5b 100644 --- a/systems/defaults.py +++ b/systems/defaults.py @@ -1,7 +1,7 @@ """ All default parameters that might be used in a system are stored here -Order of preferences is - passed in command line to calculation method, +Order of preferences is - passed in command line to calculation method, stored in system config object found in defaults @@ -9,20 +9,21 @@ from syscore.fileutils import get_filename_for_package import yaml -DEFAULT_FILENAME="systems.provided.defaults.yaml" +DEFAULT_FILENAME = "systems.provided.defaults.yaml" + def get_system_defaults(): """ >>> system_defaults['average_absolute_forecast'] 10.0 """ - default_file=get_filename_for_package(DEFAULT_FILENAME) + default_file = get_filename_for_package(DEFAULT_FILENAME) with open(default_file) as file_to_parse: - default_dict=yaml.load(file_to_parse) + default_dict = yaml.load(file_to_parse) return default_dict -system_defaults=get_system_defaults() +system_defaults = get_system_defaults() if __name__ == '__main__': import doctest diff --git a/systems/forecast_combine.py b/systems/forecast_combine.py index b3415c45a9..de2016c5a9 100644 --- a/systems/forecast_combine.py +++ b/systems/forecast_combine.py @@ -8,42 +8,43 @@ class ForecastCombineFixed(SystemStage): """ Stage for combining forecasts (already capped and scaled) - + KEY INPUT: system.forecastScaleCap.get_capped_forecast(instrument_code, rule_variation_name) found in self.get_capped_forecast(instrument_code, rule_variation_name) - + KEY OUTPUT: system.combForecast.get_combined_forecast(instrument_code) Name: combForecast """ - + def __init__(self): """ Create a SystemStage for combining forecasts - - + + """ - protected=['get_forecast_weights','get_forecast_diversification_multiplier'] + protected = ['get_forecast_weights', + 'get_forecast_diversification_multiplier'] setattr(self, "_protected", protected) setattr(self, "name", "combForecast") - + def get_capped_forecast(self, instrument_code, rule_variation_name): """ Get the capped forecast from the previous module - + KEY INPUT - - :param instrument_code: - :type str: - + + :param instrument_code: + :type str: + :param rule_variation_name: :type str: name of the trading rule variation - + :returns: dict of Tx1 pd.DataFrames; keynames rule_variation_name - + >>> from systems.tests.testdata import get_test_object_futures_with_rules_and_capping >>> from systems.basesystem import System >>> (fcs, rules, rawdata, data, config)=get_test_object_futures_with_rules_and_capping() @@ -53,27 +54,27 @@ def get_capped_forecast(self, instrument_code, rule_variation_name): 2015-04-21 5.738741 2015-04-22 5.061187 """ - - return self.parent.forecastScaleCap.get_capped_forecast(instrument_code, rule_variation_name) - - + + return self.parent.forecastScaleCap.get_capped_forecast( + instrument_code, rule_variation_name) + def get_forecast_weights(self, instrument_code): """ Get the forecast weights for this instrument - + From: (a) passed into subsystem when created (b) ... if not found then: in system.config.instrument_weights - - :param instrument_code: - :type str: - :returns: TxK pd.DataFrame containing weights, columns are trading rule variation names, T covers all + :param instrument_code: + :type str: + + :returns: TxK pd.DataFrame containing weights, columns are trading rule variation names, T covers all >>> from systems.tests.testdata import get_test_object_futures_with_rules_and_capping >>> from systems.basesystem import System >>> (fcs, rules, rawdata, data, config)=get_test_object_futures_with_rules_and_capping() >>> system=System([rawdata, rules, fcs, ForecastCombineFixed()], data, config) - >>> + >>> >>> ## from config >>> system.combForecast.get_forecast_weights("EDOLLAR").tail(2) ewmac16 ewmac8 @@ -94,63 +95,67 @@ def get_forecast_weights(self, instrument_code): ewmac16 ewmac8 2015-04-21 0.5 0.5 2015-04-22 0.5 0.5 - """ - def _get_forecast_weights(system, instrument_code, this_stage ): + """ + def _get_forecast_weights(system, instrument_code, this_stage): - ## Let's try the config + # Let's try the config if "forecast_weights" in dir(system.config): - + if instrument_code in system.config.forecast_weights: - ## nested dict - fixed_weights=system.config.forecast_weights[instrument_code] + # nested dict + fixed_weights = system.config.forecast_weights[ + instrument_code] else: - ## assume it's a non nested dict - fixed_weights=system.config.forecast_weights + # assume it's a non nested dict + fixed_weights = system.config.forecast_weights else: - rules=list(self.parent.rules.trading_rules().keys()) - weight=1.0/len(rules) - - print("WARNING: No forecast weights - using equal weights of %.4f over all %d trading rules in system" % (weight, len(rules))) - fixed_weights=dict([(rule_name, weight) for rule_name in rules]) - - - ## Now we have a dict, fixed_weights. - ## Need to turn into a timeseries covering the range of forecast dates - rule_variation_list=list(fixed_weights.keys()) - rule_variation_list.sort() - - forecasts_ts=[ - this_stage.get_capped_forecast(instrument_code, rule_variation_name).index - for rule_variation_name in rule_variation_list] - - earliest_date=min([min(fts) for fts in forecasts_ts]) - latest_date=max([max(fts) for fts in forecasts_ts]) - - ## this will be daily, but will be resampled later - weight_ts=pd.date_range(start=earliest_date, end=latest_date) - - forecasts_weights=dict([ - (rule_variation_name, pd.Series([fixed_weights[rule_variation_name]]*len(weight_ts), index=weight_ts)) - for rule_variation_name in rule_variation_list]) - - forecasts_weights=pd.concat(forecasts_weights, axis=1) - forecasts_weights.columns=rule_variation_list + rules = list(self.parent.rules.trading_rules().keys()) + weight = 1.0 / len(rules) + + print("WARNING: No forecast weights - using equal weights of %.4f over all %d trading rules in system" % + (weight, len(rules))) + fixed_weights = dict([(rule_name, weight) + for rule_name in rules]) + + # Now we have a dict, fixed_weights. + # Need to turn into a timeseries covering the range of forecast + # dates + rule_variation_list = sorted(fixed_weights.keys()) + + forecasts_ts = [ + this_stage.get_capped_forecast( + instrument_code, rule_variation_name).index + for rule_variation_name in rule_variation_list] + + earliest_date = min([min(fts) for fts in forecasts_ts]) + latest_date = max([max(fts) for fts in forecasts_ts]) + + # this will be daily, but will be resampled later + weight_ts = pd.date_range(start=earliest_date, end=latest_date) + + forecasts_weights = dict([ + (rule_variation_name, pd.Series( + [fixed_weights[rule_variation_name]] * len(weight_ts), index=weight_ts)) + for rule_variation_name in rule_variation_list]) + + forecasts_weights = pd.concat(forecasts_weights, axis=1) + forecasts_weights.columns = rule_variation_list return forecasts_weights - - forecast_weights=self.parent.calc_or_cache( "get_forecast_weights", instrument_code, _get_forecast_weights, self) - return forecast_weights + forecast_weights = self.parent.calc_or_cache( + "get_forecast_weights", instrument_code, _get_forecast_weights, self) + return forecast_weights def get_forecast_diversification_multiplier(self, instrument_code): """ - + Get the diversification multiplier for this instrument From: system.config.instrument_weights - + :param instrument_code: instrument to get multiplier for - :type instrument_code: str + :type instrument_code: str :returns: Tx1 pd.DataFrame @@ -160,7 +165,7 @@ def get_forecast_diversification_multiplier(self, instrument_code): >>> from systems.basesystem import System >>> (fcs, rules, rawdata, data, config)=get_test_object_futures_with_rules_and_capping() >>> system=System([rawdata, rules, fcs, ForecastCombineFixed()], data, config) - >>> + >>> >>> ## from config >>> system.combForecast.get_forecast_diversification_multiplier("EDOLLAR").tail(2) fdm @@ -181,54 +186,58 @@ def get_forecast_diversification_multiplier(self, instrument_code): fdm 2015-04-21 1 2015-04-22 1 - """ - def _get_forecast_div_multiplier(system, instrument_code, this_stage ): - - ## Let's try the config + """ + def _get_forecast_div_multiplier(system, instrument_code, this_stage): + + # Let's try the config if hasattr(system.config, "forecast_div_multiplier"): - if type(system.config.forecast_div_multiplier) is float: - fixed_div_mult=system.config.forecast_div_multiplier - + if isinstance(system.config.forecast_div_multiplier, float): + fixed_div_mult = system.config.forecast_div_multiplier + elif instrument_code in system.config.forecast_div_multiplier.keys(): - ## dict - fixed_div_mult=system.config.forecast_div_multiplier[instrument_code] + # dict + fixed_div_mult = system.config.forecast_div_multiplier[ + instrument_code] else: - raise Exception("FDM in config needs to be eithier float, or dict with instrument_code keys") + raise Exception( + "FDM in config needs to be eithier float, or dict with instrument_code keys") elif "forecast_div_multiplier" in system_defaults: - ## try defaults - fixed_div_mult=system_defaults['forecast_div_multiplier'] + # try defaults + fixed_div_mult = system_defaults['forecast_div_multiplier'] else: - raise Exception("Need to specify FDM in config or system_defaults") - - ## Now we have a dict, fixed_weights. - ## Need to turn into a timeseries covering the range of forecast dates - ## get forecast weights first - forecast_weights=this_stage.get_forecast_weights(instrument_code) - weight_ts=forecast_weights.index - - ts_fdm=pd.Series([fixed_div_mult]*len(weight_ts), index=weight_ts) - ts_fdm=ts_fdm.to_frame("fdm") - + raise Exception( + "Need to specify FDM in config or system_defaults") + + # Now we have a dict, fixed_weights. + # Need to turn into a timeseries covering the range of forecast dates + # get forecast weights first + forecast_weights = this_stage.get_forecast_weights(instrument_code) + weight_ts = forecast_weights.index + + ts_fdm = pd.Series([fixed_div_mult] * + len(weight_ts), index=weight_ts) + ts_fdm = ts_fdm.to_frame("fdm") + return ts_fdm - - forecast_div_multiplier=self.parent.calc_or_cache( 'get_forecast_diversification_multiplier', instrument_code, _get_forecast_div_multiplier, self) + + forecast_div_multiplier = self.parent.calc_or_cache( + 'get_forecast_diversification_multiplier', instrument_code, _get_forecast_div_multiplier, self) return forecast_div_multiplier - - + def get_combined_forecast(self, instrument_code): """ Get a combined forecast, linear combination of individual forecasts with FDM applied - + We forward fill all forecasts. We then adjust forecast weights so that they are 1.0 in every period; after setting to zero when no forecast is available. Finally we multiply up, and apply the FDM. - :param instrument_code: - :type str: - + :param instrument_code: + :type str: + :returns: Tx1 pd.DataFrame - + KEY OUTPUT @@ -236,41 +245,46 @@ def get_combined_forecast(self, instrument_code): >>> from systems.basesystem import System >>> (fcs, rules, rawdata, data, config)=get_test_object_futures_with_rules_and_capping() >>> system=System([rawdata, rules, fcs, ForecastCombineFixed()], data, config) - >>> + >>> >>> system.combForecast.get_combined_forecast("EDOLLAR").tail(2) comb_forecast 2015-04-21 7.622781 2015-04-22 6.722785 - """ - def _get_combined_forecast(system, instrument_code, this_stage ): - - forecast_weights=this_stage.get_forecast_weights(instrument_code) - rule_variation_list=list(forecast_weights.columns) - forecasts=[this_stage.get_capped_forecast(instrument_code, rule_variation_name) for rule_variation_name in rule_variation_list] - forecast_div_multiplier=this_stage.get_forecast_diversification_multiplier(instrument_code) - - forecasts=pd.concat(forecasts, axis=1) - - ## adjust weights for missing data - forecast_weights=fix_weights_vs_pdm(forecast_weights, forecasts) - - ## multiply weights by forecasts - - combined_forecast=multiply_df(forecast_weights, forecasts) - - ## sum - combined_forecast=combined_forecast.sum(axis=1).to_frame("comb_forecast") - - ## apply fdm - ## (note in this simple version we aren't adjusting FDM if forecast_weights change) - forecast_div_multiplier=forecast_div_multiplier.reindex(forecasts.index, method="ffill") - combined_forecast=multiply_df(combined_forecast,forecast_div_multiplier) - + """ + def _get_combined_forecast(system, instrument_code, this_stage): + + forecast_weights = this_stage.get_forecast_weights(instrument_code) + rule_variation_list = list(forecast_weights.columns) + forecasts = [this_stage.get_capped_forecast( + instrument_code, rule_variation_name) for rule_variation_name in rule_variation_list] + forecast_div_multiplier = this_stage.get_forecast_diversification_multiplier( + instrument_code) + + forecasts = pd.concat(forecasts, axis=1) + + # adjust weights for missing data + forecast_weights = fix_weights_vs_pdm(forecast_weights, forecasts) + + # multiply weights by forecasts + + combined_forecast = multiply_df(forecast_weights, forecasts) + + # sum + combined_forecast = combined_forecast.sum( + axis=1).to_frame("comb_forecast") + + # apply fdm + # (note in this simple version we aren't adjusting FDM if forecast_weights change) + forecast_div_multiplier = forecast_div_multiplier.reindex( + forecasts.index, method="ffill") + combined_forecast = multiply_df( + combined_forecast, forecast_div_multiplier) + return combined_forecast - - combined_forecast=self.parent.calc_or_cache( 'get_combined_forecast', instrument_code, _get_combined_forecast, self) - return combined_forecast + combined_forecast = self.parent.calc_or_cache( + 'get_combined_forecast', instrument_code, _get_combined_forecast, self) + return combined_forecast if __name__ == '__main__': diff --git a/systems/forecast_scale_cap.py b/systems/forecast_scale_cap.py index e2161c2b0c..f7de433359 100644 --- a/systems/forecast_scale_cap.py +++ b/systems/forecast_scale_cap.py @@ -3,50 +3,50 @@ from systems.defaults import system_defaults from syscore.pdutils import apply_cap + class ForecastScaleCapFixed(SystemStage): """ Create a SystemStage for scaling and capping forecasting - + This simple variation uses Fixed capping and scaling - + KEY INPUT: system.rules.get_raw_forecast(instrument_code, rule_variation_name) found in self.get_raw_forecast(instrument_code, rule_variation_name) - + KEY OUTPUT: system.forecastScaleCap.get_capped_forecast(instrument_code, rule_variation_name) Name: forecastScaleCap """ - - + def __init__(self): """ Create a SystemStage for scaling and capping forecasting - + Using Fixed capping and scaling - + :returns: None - + """ - protected=["get_forecast_scalars"] + protected = ["get_forecast_scalars"] setattr(self, "_protected", protected) setattr(self, "name", "forecastScaleCap") - + def get_raw_forecast(self, instrument_code, rule_variation_name): """ Convenience method as we use the raw forecast several times - + KEY_INPUT - :param instrument_code: - :type str: - + :param instrument_code: + :type str: + :param rule_variation_name: :type str: name of the trading rule variation - + :returns: Tx1 pd.DataFrame, same size as forecast - + >>> from systems.tests.testdata import get_test_object_futures_with_rules >>> from systems.basesystem import System >>> (rules, rawdata, data, config)=get_test_object_futures_with_rules() @@ -57,26 +57,26 @@ def get_raw_forecast(self, instrument_code, rule_variation_name): 2015-04-22 0.954941 """ - raw_forecast=self.parent.rules.get_raw_forecast(instrument_code, rule_variation_name) - + raw_forecast = self.parent.rules.get_raw_forecast( + instrument_code, rule_variation_name) + return raw_forecast - - + def get_forecast_scalar(self, instrument_code, rule_variation_name): """ Get the scalar to apply to raw forecasts - + In this simple version it's the same for all instruments, and fixed We get the scalars from: (a) configuration file in parent system (b) or if missing: uses the scalar from systems.defaults.py - :param instrument_code: - :type str: - + :param instrument_code: + :type str: + :param rule_variation_name: :type str: name of the trading rule variation - + :returns: float >>> from systems.tests.testdata import get_test_object_futures_with_rules @@ -100,41 +100,45 @@ def get_forecast_scalar(self, instrument_code, rule_variation_name): >>> system4.forecastScaleCap.get_forecast_scalar("EDOLLAR", "ewmac8") 11.0 """ - - def _get_forecast_scalar(system, instrument_code, rule_variation_name, this_stage): - ## Try the config file + + def _get_forecast_scalar( + system, instrument_code, rule_variation_name, this_stage): + # Try the config file try: - scalar=system.config.trading_rules[rule_variation_name]['forecast_scalar'] + scalar = system.config.trading_rules[ + rule_variation_name]['forecast_scalar'] except: try: - ## can also put somewhere else ... - scalar=system.config.forecast_scalars[rule_variation_name] + # can also put somewhere else ... + scalar = system.config.forecast_scalars[ + rule_variation_name] except: - ## go with defaults - scalar=system_defaults['forecast_scalar'] - + # go with defaults + scalar = system_defaults['forecast_scalar'] + return scalar - - forecast_scalar=self.parent.calc_or_cache_nested( "get_forecast_scalar", instrument_code, rule_variation_name, _get_forecast_scalar, self) + + forecast_scalar = self.parent.calc_or_cache_nested( + "get_forecast_scalar", instrument_code, rule_variation_name, _get_forecast_scalar, self) return forecast_scalar - + def get_forecast_cap(self, instrument_code, rule_variation_name): """ Get forecast cap - + In this simple version it's the same for all instruments, and rule variations - - We get the cap from: + + We get the cap from: (a) configuration object in parent system (c) or if missing: uses the forecast_cap from systems.default.py - - :param instrument_code: - :type str: - + + :param instrument_code: + :type str: + :param rule_variation_name: :type str: name of the trading rule variation - + :returns: float >>> from systems.tests.testdata import get_test_object_futures_with_rules @@ -154,31 +158,32 @@ def get_forecast_cap(self, instrument_code, rule_variation_name): """ - def _get_forecast_cap(system, instrument_code, rule_variation_name, this_stage): - ## Try the config file + def _get_forecast_cap(system, instrument_code, + rule_variation_name, this_stage): + # Try the config file try: - cap=system.config.forecast_cap + cap = system.config.forecast_cap except: - ## go with defaults - cap=system_defaults['forecast_cap'] - + # go with defaults + cap = system_defaults['forecast_cap'] + return cap - - forecast_cap=self.parent.calc_or_cache_nested( "get_forecast_cap", instrument_code, rule_variation_name, _get_forecast_cap, self) + + forecast_cap = self.parent.calc_or_cache_nested( + "get_forecast_cap", instrument_code, rule_variation_name, _get_forecast_cap, self) return forecast_cap - def get_scaled_forecast(self, instrument_code, rule_variation_name): """ Return the scaled forecast - - :param instrument_code: - :type str: - + + :param instrument_code: + :type str: + :param rule_variation_name: :type str: name of the trading rule variation - + :returns: Tx1 pd.DataFrame, same size as forecast >>> from systems.tests.testdata import get_test_object_futures_with_rules @@ -190,16 +195,20 @@ def get_scaled_forecast(self, instrument_code, rule_variation_name): 2015-04-21 5.738741 2015-04-22 5.061187 """ - - def _get_scaled_forecast(system, instrument_code, rule_variation_name, this_stage): - raw_forecast=this_stage.get_raw_forecast(instrument_code, rule_variation_name) - scale=this_stage.get_forecast_scalar(instrument_code, rule_variation_name) - - scaled_forecast=raw_forecast*scale - + + def _get_scaled_forecast( + system, instrument_code, rule_variation_name, this_stage): + raw_forecast = this_stage.get_raw_forecast( + instrument_code, rule_variation_name) + scale = this_stage.get_forecast_scalar( + instrument_code, rule_variation_name) + + scaled_forecast = raw_forecast * scale + return scaled_forecast - - scaled_forecast=self.parent.calc_or_cache_nested( "get_scaled_forecast", instrument_code, rule_variation_name, _get_scaled_forecast, self) + + scaled_forecast = self.parent.calc_or_cache_nested( + "get_scaled_forecast", instrument_code, rule_variation_name, _get_scaled_forecast, self) return scaled_forecast @@ -210,13 +219,13 @@ def get_capped_forecast(self, instrument_code, rule_variation_name): KEY OUTPUT - - :param instrument_code: - :type str: - + + :param instrument_code: + :type str: + :param rule_variation_name: :type str: name of the trading rule variation - + :returns: Tx1 pd.DataFrame, same size as forecast >>> from systems.tests.testdata import get_test_object_futures_with_rules @@ -228,25 +237,29 @@ def get_capped_forecast(self, instrument_code, rule_variation_name): ewmac8 2015-04-21 4 2015-04-22 4 - - + + """ - - def _get_capped_forecast(system, instrument_code, rule_variation_name, this_stage): - - scaled_forecast=this_stage.get_scaled_forecast(instrument_code, rule_variation_name) - cap=this_stage.get_forecast_cap(instrument_code, rule_variation_name) - - capped_forecast=apply_cap(scaled_forecast, cap) - capped_forecast.columns=scaled_forecast.columns - + + def _get_capped_forecast( + system, instrument_code, rule_variation_name, this_stage): + + scaled_forecast = this_stage.get_scaled_forecast( + instrument_code, rule_variation_name) + cap = this_stage.get_forecast_cap( + instrument_code, rule_variation_name) + + capped_forecast = apply_cap(scaled_forecast, cap) + capped_forecast.columns = scaled_forecast.columns + return capped_forecast - - capped_forecast=self.parent.calc_or_cache_nested( "get_capped_forecast", instrument_code, rule_variation_name, _get_capped_forecast, self) + + capped_forecast = self.parent.calc_or_cache_nested( + "get_capped_forecast", instrument_code, rule_variation_name, _get_capped_forecast, self) return capped_forecast - + if __name__ == '__main__': import doctest doctest.testmod() diff --git a/systems/forecasting.py b/systems/forecasting.py index 4c16586253..7122798e66 100644 --- a/systems/forecasting.py +++ b/systems/forecasting.py @@ -3,251 +3,262 @@ from systems.stage import SystemStage from syscore.objects import resolve_function, resolve_data_method, hasallattr + class Rules(SystemStage): """ Construct the forecasting stage - + Ways we can do this: - + a) We do this by passing a list of trading rules - + forecasting([trading_rule1, trading_rule2, ..]) - + Note that trading rules can be created using the TradingRule class, or you can just pass in the - name of a function, or a function. - + name of a function, or a function. + We can also use a generate_variations method to create a list of multiple rules - + b) or we can create from a system config - - - KEY INPUT: Depends on trading rule(s) data argument + + + KEY INPUT: Depends on trading rule(s) data argument KEY OUTPUT: system.rules.get_raw_forecast(instrument_code, rule_variation_name) system.rules.trading_rules() - + Name: rules """ - + def __init__(self, trading_rules=None): """ Create a SystemStage for forecasting - + We eithier pass a dict or a list of trading rules (functions, strings specifying a function, or objects of class TradingRule) ... or we'll get it from the overall system config (trading_rules=None) - - :param trading_rules: Set of trading rules + + :param trading_rules: Set of trading rules :type trading_rules: None (rules will be inherited from self.parent system) TradingRule, str, callable function, or tuple (single rule) - list or dict (multiple rules) + list or dict (multiple rules) :returns: Rules object - + """ setattr(self, "name", "rules") - - ## We won't have trading rules we can use until we've parsed them + + # We won't have trading rules we can use until we've parsed them setattr(self, "_trading_rules", None) - ## ... store the ones we've been passed for now + # ... store the ones we've been passed for now setattr(self, "_passed_trading_rules", trading_rules) def __repr__(self): - trading_rules=self._trading_rules - + trading_rules = self._trading_rules + if trading_rules is not None: - rule_names=", ".join(self._trading_rules.keys()) - return "Rules object with rules "+rule_names + rule_names = ", ".join(self._trading_rules.keys()) + return "Rules object with rules " + rule_names else: return "Rules object with unknown trading rules [try Rules.tradingrules() ]" - + def trading_rules(self): """ Ensure self.trading_rules is actually a properly specified list of trading rules - + We can't do this when we __init__ because we might not have a parent yet :returns: List of TradingRule objects - + """ - - current_rules=self._trading_rules - ## We have already parsed the trading rules for this object, just return them + current_rules = self._trading_rules + + # We have already parsed the trading rules for this object, just return + # them if current_rules is not None: return current_rules - - ## What where we passed when object was created? - passed_rules=self._passed_trading_rules - + + # What where we passed when object was created? + passed_rules = self._passed_trading_rules + if passed_rules is None: """ We weren't passed anything in the command lines so need to inherit from the system config """ - + if not hasattr(self, "parent"): - raise Exception("A Rules stage needs to be part of a System to identify trading rules, unless rules are passed when object created") - + raise Exception( + "A Rules stage needs to be part of a System to identify trading rules, unless rules are passed when object created") + if not hasattr(self.parent, "config"): - raise Exception("A system needs to include a config with trading_rules, unless rules are passed when object created") - + raise Exception( + "A system needs to include a config with trading_rules, unless rules are passed when object created") + if not hasattr(self.parent.config, "trading_rules"): - raise Exception("A system config needs to include trading_rules, unless rules are passed when object created") - - ## self.parent.config.tradingrules will already be in dictionary form - forecasting_config=self.parent.config.trading_rules - new_rules=process_trading_rules(forecasting_config) - + raise Exception( + "A system config needs to include trading_rules, unless rules are passed when object created") + + # self.parent.config.tradingrules will already be in dictionary + # form + forecasting_config = self.parent.config.trading_rules + new_rules = process_trading_rules(forecasting_config) + else: - ### Okay, we've been passed a list manually which we'll use rather than getting it from the system - new_rules=process_trading_rules(passed_rules) - + # Okay, we've been passed a list manually which we'll use rather + # than getting it from the system + new_rules = process_trading_rules(passed_rules) + setattr(self, "_trading_rules", new_rules) return(new_rules) def get_raw_forecast(self, instrument_code, rule_variation_name): """ Does what it says on the tin - pulls the forecast for the trading rule - + This forecast will need scaling and capping later - + KEY OUTPUT - + """ - - def _get_forecast(system, instrument_code, rule_variation_name, rules_stage): - ## This function gets called if we haven't cached the forecast - trading_rule=rules_stage.trading_rules()[rule_variation_name] - - result=trading_rule.call(system, instrument_code) - result.columns=[rule_variation_name] - + + def _get_forecast(system, instrument_code, + rule_variation_name, rules_stage): + # This function gets called if we haven't cached the forecast + trading_rule = rules_stage.trading_rules()[rule_variation_name] + + result = trading_rule.call(system, instrument_code) + result.columns = [rule_variation_name] + return result - - forecast=self.parent.calc_or_cache_nested( "get_raw_forecast", instrument_code, rule_variation_name, _get_forecast, self) - return forecast + forecast = self.parent.calc_or_cache_nested( + "get_raw_forecast", instrument_code, rule_variation_name, _get_forecast, self) + return forecast class TradingRule(object): """ Container for trading rules - + Can be called manually or will be called when configuring a system """ - - + def __init__(self, rule, data=list(), other_args=dict()): """ Create a trading rule from a function Functions must be of the form function(*dataargs, **kwargs), where *dataargs are unnamed data items, and **kwargs are named configuration items - + data, an ordered list of strings identifying data to be used (default, just price) other_args: a dictionary of named arguments to be passed to the trading rule - :param rule: Trading rule to be created - :type trading_rules: + :param rule: Trading rule to be created + :type trading_rules: The following describe a rule completely (ignore data and other_args arguments) - 3-tuple ; containing (function, data, other_args) - dict (containing key "function", and optionally keys "other_args" and "data") + 3-tuple ; containing (function, data, other_args) + dict (containing key "function", and optionally keys "other_args" and "data") TradingRule (object is created out of this rule) - + The following will be combined with the data and other_args arguments to produce a complete TradingRule: - - Other callable function + + Other callable function str (with path to function eg "systems.provide.example.rules.ewmac_forecast_with_defaults") :param data: (list of) str pointing to location of inputs in a system method call (eg "data.get_instrument_price") (Eithier passed in separately, or as part of a TradingRule, 3-tuple, or dict object) :type data: single str, or list of str - + :param other_args: Other named arguments to be passed to trading rule function (Eithier passed in separately , or as part of a TradingRule, 3-tuple, or dict object) :type other_args: dict - + :returns: single Tradingrule object - + """ if hasallattr(rule, ["function", "data", "other_args"]): - ## looks like it is already a trading rule - (rule_function, data, other_args)=(rule.function, rule.data, rule.other_args) - - - elif type(rule) is tuple: - if len(data)>0 or len(other_args)>0: - print("WARNING: Creating trade rule with 'rule' tuple argument, ignoring data and/or other args") - - if len(rule)!=3: - raise Exception("Creating trading rule with a tuple, must be length 3 exactly (function/name, data [...], args dict(...))") - (rule_function, data, other_args)=rule - - elif type(rule) is dict: - if len(data)>0 or len(other_args)>0: - print("WARNING: Creating trade rule with 'rule' dict argument, ignoring data and/or other args") + # looks like it is already a trading rule + (rule_function, data, other_args) = ( + rule.function, rule.data, rule.other_args) + + elif isinstance(rule, tuple): + if len(data) > 0 or len(other_args) > 0: + print( + "WARNING: Creating trade rule with 'rule' tuple argument, ignoring data and/or other args") + + if len(rule) != 3: + raise Exception( + "Creating trading rule with a tuple, must be length 3 exactly (function/name, data [...], args dict(...))") + (rule_function, data, other_args) = rule + + elif isinstance(rule, dict): + if len(data) > 0 or len(other_args) > 0: + print( + "WARNING: Creating trade rule with 'rule' dict argument, ignoring data and/or other args") try: - rule_function=rule['function'] + rule_function = rule['function'] except KeyError: - raise Exception("If you specify a TradingRule as a dict it has to contain a 'function' keyname") - + raise Exception( + "If you specify a TradingRule as a dict it has to contain a 'function' keyname") + if "data" in rule: - data=rule['data'] + data = rule['data'] else: - data=[] - + data = [] + if "other_args" in rule: - other_args=rule['other_args'] - + other_args = rule['other_args'] + else: - other_args=dict() + other_args = dict() else: - rule_function=rule + rule_function = rule - ## turn string into a callable function if required - rule_function=resolve_function(rule_function) + # turn string into a callable function if required + rule_function = resolve_function(rule_function) + + if isinstance(data, str): + # turn into a 1 item list or wont' get parsed properly + data = [data] - if type(data) is str: - ## turn into a 1 item list or wont' get parsed properly - data=[data] - setattr(self, "function", rule_function) setattr(self, "data", data) setattr(self, "other_args", other_args) - + def __repr__(self): - data_names=", ".join(self.data) - args_names=", ".join(self.other_args.keys()) - return "TradingRule; function: %s, data: %s and other_args: %s" % (str(self.function), data_names, args_names) - + data_names = ", ".join(self.data) + args_names = ", ".join(self.other_args.keys()) + return "TradingRule; function: %s, data: %s and other_args: %s" % ( + str(self.function), data_names, args_names) + def call(self, system, instrument_code): """ Actually call a trading rule - + To do this we need some data from the system """ - - assert type(self.data) is list - - if len(self.data)==0: - ## if no data provided defaults to using price - datalist=["data.get_instrument_price"] + + assert isinstance(self.data, list) + + if len(self.data) == 0: + # if no data provided defaults to using price + datalist = ["data.get_instrument_price"] else: - datalist=self.data - - data_methods=[resolve_data_method(system, data_string) for data_string in datalist] - data=[data_method(instrument_code) for data_method in data_methods] - - other_args=self.other_args - + datalist = self.data + + data_methods = [resolve_data_method( + system, data_string) for data_string in datalist] + data = [data_method(instrument_code) for data_method in data_methods] + + other_args = self.other_args + return self.function(*data, **other_args) - - def process_trading_rules(trading_rules): @@ -255,59 +266,63 @@ def process_trading_rules(trading_rules): There are a number of ways to specify a set of trading rules. This function processes them all, and returns a dict of TradingRule objects. - - data types handled: + + data types handled: dict - parse each element of the dict and use the names [unless has one or more of keynames: function, data, args] list - parse each element of the list and give them arbitrary names anything else is assumed to be something we can pass to TradingRule (string, function, tuple, (dict with keynames function, data, args), or TradingRule object) - :param trading_rules: Set of trading rules - + :param trading_rules: Set of trading rules + :type trading_rules: Single rule: - dict(function=str, optionally: args=dict(), optionally: data=list()), - TradingRule, str, callable function, or tuple - + dict(function=str, optionally: args=dict(), optionally: data=list()), + TradingRule, str, callable function, or tuple + Multiple rules: - list, dict without 'function' keyname + list, dict without 'function' keyname :returns: dict of Tradingrule objects - + """ - if type(trading_rules) is list: - ## Give some arbitrary name - ans=dict([("rule%d" % ruleid, TradingRule(rule)) for (ruleid, rule) in enumerate(trading_rules)]) + if isinstance(trading_rules, list): + # Give some arbitrary name + ans = dict([("rule%d" % ruleid, TradingRule(rule)) + for (ruleid, rule) in enumerate(trading_rules)]) return ans - if type(trading_rules) is dict: + if isinstance(trading_rules, dict): if "function" not in trading_rules: - ## Note the system config will always come in as a dict - ans=dict([(keyname, TradingRule(trading_rules[keyname])) for keyname in trading_rules]) + # Note the system config will always come in as a dict + ans = dict([(keyname, TradingRule(trading_rules[keyname])) + for keyname in trading_rules]) return ans - - ## Must be an individual rule (string, function, dict with 'function' or tuple) + + # Must be an individual rule (string, function, dict with 'function' or + # tuple) return process_trading_rules([trading_rules]) -def create_variations_oneparameter(baseRule, list_of_args, argname, nameformat="%s_%s"): +def create_variations_oneparameter( + baseRule, list_of_args, argname, nameformat="%s_%s"): """ Returns a dict of trading rule variations, varying only one named parameter - - :param baseRule: Trading rule to copy + + :param baseRule: Trading rule to copy :type baseRule: TradingRule object - + :param list_of_args: set of parameters to use :type list_of_args: list - + :param argname: Argument passed to trading rule which will be changed :type argname: str - + :param nameformat: Format to use when naming trading rules; nameformat % (argname, argvalue) will be used :type nameformat: str containing two '%s' elements - + :returns: dict of Tradingrule objects - - >>> + + >>> >>> rule=TradingRule(("systems.provided.example.rules.ewmac_forecast_with_defaults", [], {})) >>> variations=create_variations_oneparameter(rule, [4,10,100], "Lfast") >>> ans=list(variations.keys()) @@ -315,37 +330,40 @@ def create_variations_oneparameter(baseRule, list_of_args, argname, nameformat=" >>> ans ['Lfast_10', 'Lfast_100', 'Lfast_4'] """ - list_of_args_dict=[] + list_of_args_dict = [] for arg_value in list_of_args: - thisdict=dict() - thisdict[argname]=arg_value + thisdict = dict() + thisdict[argname] = arg_value list_of_args_dict.append(thisdict) - ans=create_variations(baseRule, list_of_args_dict, key_argname=argname, nameformat=nameformat) - + ans = create_variations(baseRule, list_of_args_dict, + key_argname=argname, nameformat=nameformat) + return ans -def create_variations(baseRule, list_of_args_dict, key_argname=None, nameformat="%s_%s"): + +def create_variations(baseRule, list_of_args_dict, + key_argname=None, nameformat="%s_%s"): """ - Returns a dict of trading rule variations - + Returns a dict of trading rule variations + eg create_variations(ewmacrule, [dict(fast=2, slow=8), dict(fast=4, ...) ], argname="fast", basename="ewmac") - :param baseRule: Trading rule to copy + :param baseRule: Trading rule to copy :type baseRule: TradingRule object - - :param list_of_args_dict: sets of parameters to use. + + :param list_of_args_dict: sets of parameters to use. :type list_of_args: list of dicts; each dict contains a set of parameters to vary for each instance - + :param key_argname: Non :type key_argname: str or None (None is allowed if only one parameter is changed) - + :param nameformat: Format to use when naming trading rules; nameformat % (argname, argvalue) will be used :type nameformat: str containing two '%s' elements - + :returns: dict of Tradingrule objects - + >>> rule=TradingRule(("systems.provided.example.rules.ewmac_forecast_with_defaults", [], {})) >>> variations=create_variations(rule, [dict(Lfast=2, Lslow=8), dict(Lfast=4, Lslow=16)], "Lfast", nameformat="ewmac_%s_%s") >>> ans=list(variations.keys()) @@ -353,36 +371,38 @@ def create_variations(baseRule, list_of_args_dict, key_argname=None, nameforma >>> ans ['ewmac_Lfast_2', 'ewmac_Lfast_4'] """ - + if key_argname is None: - - if all([len(args_dict)==1 for args_dict in list_of_args_dict]): - ## okay to use argname as only seems to be one of them - key_argname=args_dict[0].keys()[0] + + if all([len(args_dict) == 1 for args_dict in list_of_args_dict]): + # okay to use argname as only seems to be one of them + key_argname = args_dict[0].keys()[0] else: - raise Exception("need to specify argname if more than one possibility") - - - baseRulefunction=baseRule.function - baseRuledata=baseRule.data - - ## these will be overwritten as we run through - baseRuleargs=copy(baseRule.other_args) - - variations=dict() - + raise Exception( + "need to specify argname if more than one possibility") + + baseRulefunction = baseRule.function + baseRuledata = baseRule.data + + # these will be overwritten as we run through + baseRuleargs = copy(baseRule.other_args) + + variations = dict() + for args_dict in list_of_args_dict: if key_argname not in args_dict.keys(): - raise Exception("Argname %s missing from at least one set of argument values" % key_argname) - + raise Exception( + "Argname %s missing from at least one set of argument values" % key_argname) + for arg_name in args_dict.keys(): - baseRuleargs[arg_name]=args_dict[arg_name] - - rule_variation=TradingRule(baseRulefunction, baseRuledata, baseRuleargs) - var_name=nameformat % (key_argname, str(args_dict[key_argname])) - - variations[var_name]=rule_variation - + baseRuleargs[arg_name] = args_dict[arg_name] + + rule_variation = TradingRule( + baseRulefunction, baseRuledata, baseRuleargs) + var_name = nameformat % (key_argname, str(args_dict[key_argname])) + + variations[var_name] = rule_variation + return variations diff --git a/systems/futures/rawdata.py b/systems/futures/rawdata.py index dbdc2e89e6..87b33e5d5e 100644 --- a/systems/futures/rawdata.py +++ b/systems/futures/rawdata.py @@ -10,45 +10,42 @@ class FuturesRawData(RawData): """ A SubSystem that does futures specific raw data calculations - KEY INPUT: system.data.get_instrument_raw_carry_data(instrument_code) + KEY INPUT: system.data.get_instrument_raw_carry_data(instrument_code) found in self.get_instrument_raw_carry_data(self, instrument_code) - + KEY OUTPUT: system.rawdata.daily_annualised_roll(instrument_code) Name: rawdata """ - - def __init__(self): - """ Create a futures raw data subsystem - + >>> FuturesRawData() SystemStage 'rawdata' """ - + super(FuturesRawData, self).__init__() """ if you add another method to this you also need to add its blank dict here """ - - protected=[] - update_recalc(self, protected) - + + protected = [] + update_recalc(self, protected) + def get_instrument_raw_carry_data(self, instrument_code): """ Returns the 4 columns PRICE, CARRY, PRICE_CONTRACT, CARRY_CONTRACT - + :param instrument_code: instrument to get data for :type instrument_code: str - + :returns: Tx4 pd.DataFrame - + KEY INPUT - + >>> from systems.tests.testdata import get_test_object_futures >>> from systems.basesystem import System >>> (rawdata, data, config)=get_test_object_futures() @@ -58,25 +55,24 @@ def get_instrument_raw_carry_data(self, instrument_code): 2015-04-21 97.83 97.9050 201806 201809 2015-04-22 NaN 97.8325 201806 201809 """ - + def _calc_raw_carry(system, instrument_code): - instrcarrydata=system.data.get_instrument_raw_carry_data(instrument_code) + instrcarrydata = system.data.get_instrument_raw_carry_data( + instrument_code) return instrcarrydata - raw_carry=self.parent.calc_or_cache( "instrument_raw_carry_data", instrument_code, _calc_raw_carry) - + raw_carry = self.parent.calc_or_cache( + "instrument_raw_carry_data", instrument_code, _calc_raw_carry) + return raw_carry - - - def raw_futures_roll(self, instrument_code): """ Returns the raw difference between price and carry :param instrument_code: instrument to get data for :type instrument_code: str - + :returns: Tx4 pd.DataFrame >>> from systems.tests.testdata import get_test_object_futures @@ -89,29 +85,29 @@ def raw_futures_roll(self, instrument_code): 2015-04-22 NaN """ - def _calc_raw_futures_roll(system, instrument_code, this_subsystem): - - carrydata=this_subsystem.get_instrument_raw_carry_data(instrument_code) - raw_roll=carrydata.PRICE - carrydata.CARRY - - raw_roll[raw_roll==0]=np.nan - raw_roll=raw_roll.to_frame('raw_roll') + carrydata = this_subsystem.get_instrument_raw_carry_data( + instrument_code) + raw_roll = carrydata.PRICE - carrydata.CARRY + + raw_roll[raw_roll == 0] = np.nan + + raw_roll = raw_roll.to_frame('raw_roll') return raw_roll - raw_roll=self.parent.calc_or_cache( "raw_futures_roll", instrument_code, _calc_raw_futures_roll, self) - + raw_roll = self.parent.calc_or_cache( + "raw_futures_roll", instrument_code, _calc_raw_futures_roll, self) + return raw_roll - def roll_differentials(self, instrument_code): """ - Work out the annualisation factor + Work out the annualisation factor :param instrument_code: instrument to get data for :type instrument_code: str - + :returns: Tx4 pd.DataFrame >>> from systems.tests.testdata import get_test_object_futures @@ -124,25 +120,26 @@ def roll_differentials(self, instrument_code): 2015-04-22 -0.251882 """ def _calc_roll_differentials(system, instrument_code, this_subsystem): - carrydata=this_subsystem.get_instrument_raw_carry_data(instrument_code) - roll_diff=carrydata.apply(expiry_diff, 1) + carrydata = this_subsystem.get_instrument_raw_carry_data( + instrument_code) + roll_diff = carrydata.apply(expiry_diff, 1) + + roll_diff = roll_diff.to_frame('roll_diff') - roll_diff=roll_diff.to_frame('roll_diff') - return roll_diff - - roll_diff=self.parent.calc_or_cache( "roll_differentials", instrument_code, _calc_roll_differentials, self) - + + roll_diff = self.parent.calc_or_cache( + "roll_differentials", instrument_code, _calc_roll_differentials, self) + return roll_diff - def annualised_roll(self, instrument_code): """ Work out annualised futures roll - + :param instrument_code: instrument to get data for :type instrument_code: str - + :returns: Tx4 pd.DataFrame >>> from systems.tests.testdata import get_test_object_futures @@ -155,34 +152,34 @@ def annualised_roll(self, instrument_code): 2015-04-22 NaN """ - + def _calc_annualised_roll(system, instrument_code, this_subsystem): - rolldiffs=this_subsystem.roll_differentials(instrument_code) - rawrollvalues=this_subsystem.raw_futures_roll(instrument_code) + rolldiffs = this_subsystem.roll_differentials(instrument_code) + rawrollvalues = this_subsystem.raw_futures_roll(instrument_code) - annroll=divide_df_single_column(rawrollvalues, rolldiffs) - annroll.columns=['annualised_roll'] + annroll = divide_df_single_column(rawrollvalues, rolldiffs) + annroll.columns = ['annualised_roll'] return annroll - annroll=self.parent.calc_or_cache( "annualised_roll", instrument_code, _calc_annualised_roll, self) + annroll = self.parent.calc_or_cache( + "annualised_roll", instrument_code, _calc_annualised_roll, self) return annroll - - + def daily_annualised_roll(self, instrument_code): """ Resample annualised roll to daily frequency - + We don't resample earlier, or we'll get bad data - + :param instrument_code: instrument to get data for :type instrument_code: str - + :returns: Tx4 pd.DataFrame KEY OUTPUT - + >>> from systems.tests.testdata import get_test_object_futures >>> from systems.basesystem import System >>> (rawdata, data, config)=get_test_object_futures() @@ -192,28 +189,27 @@ def daily_annualised_roll(self, instrument_code): 2015-04-21 0.297758 2015-04-22 NaN """ - + def _calc_daily_ann_roll(system, instrument_code, this_subsystem): - - annroll=this_subsystem.annualised_roll(instrument_code) - annroll=annroll.resample("1B", how="mean") - annroll.columns=['annualised_roll_daily'] + + annroll = this_subsystem.annualised_roll(instrument_code) + annroll = annroll.resample("1B", how="mean") + annroll.columns = ['annualised_roll_daily'] return annroll - - - ann_daily_roll=self.parent.calc_or_cache( "daily_annualised_roll", instrument_code, _calc_daily_ann_roll, self) - + + ann_daily_roll = self.parent.calc_or_cache( + "daily_annualised_roll", instrument_code, _calc_daily_ann_roll, self) + return ann_daily_roll - - + def daily_denominator_price(self, instrument_code): """ Gets daily prices for use with % volatility This won't always be the same as the normal 'price' - :param instrument_code: Instrument to get prices for + :param instrument_code: Instrument to get prices for :type trading_rules: str - + :returns: Tx1 pd.DataFrame KEY OUTPUT @@ -230,18 +226,18 @@ def daily_denominator_price(self, instrument_code): """ def _daily_denominator_prices(system, instrument_code, this_subsystem): - prices=this_subsystem.get_instrument_raw_carry_data( instrument_code).PRICE.to_frame() - daily_prices=prices.resample("1B", how="last") - daily_prices.columns=['price'] + prices = this_subsystem.get_instrument_raw_carry_data( + instrument_code).PRICE.to_frame() + daily_prices = prices.resample("1B", how="last") + daily_prices.columns = ['price'] return daily_prices - - daily_dem_prices=self.parent.calc_or_cache( "daily_denominator_price", instrument_code, _daily_denominator_prices, self) - + + daily_dem_prices = self.parent.calc_or_cache( + "daily_denominator_price", instrument_code, _daily_denominator_prices, self) + return daily_dem_prices - - if __name__ == '__main__': import doctest - doctest.testmod() \ No newline at end of file + doctest.testmod() diff --git a/systems/portfolio.py b/systems/portfolio.py index 0fd9bf77b7..f759c9cd66 100644 --- a/systems/portfolio.py +++ b/systems/portfolio.py @@ -2,57 +2,58 @@ from systems.stage import SystemStage from systems.basesystem import ALL_KEYNAME -from syscore.pdutils import multiply_df_single_column, fix_weights_vs_pdm +from syscore.pdutils import multiply_df_single_column, fix_weights_vs_pdm from systems.defaults import system_defaults + class PortfoliosFixed(SystemStage): """ - Stage for portfolios - + Stage for portfolios + Gets the position, accounts for instrument weights and diversification multiplier - + This version involves fixed weights and multipliers. - + Note: At this stage we're dealing with a notional, fixed, amount of capital. We'll need to work out p&l to scale positions properly - + KEY INPUT: system.positionSize.get_subsystem_position(instrument_code) found in self.get_subsystem_position(instrument_code) - - KEY OUTPUT: system.portfolio.get_notional_position(instrument_code) + + KEY OUTPUT: system.portfolio.get_notional_position(instrument_code) Name: portfolio """ - + def __init__(self): """ Create a SystemStage for creating portfolios - - + + """ - protected=["get_instrument_weights", "get_instrument_diversification_multiplier", "get_raw_instrument_weights"] - + protected = ["get_instrument_weights", + "get_instrument_diversification_multiplier", "get_raw_instrument_weights"] + setattr(self, "_protected", protected) setattr(self, "name", "portfolio") - def get_subsystem_position(self, instrument_code): """ Get the position assuming all capital in one position, from a previous module - + :param instrument_code: instrument to get values for :type instrument_code: str - - :returns: Tx1 pd.DataFrame - + + :returns: Tx1 pd.DataFrame + KEY INPUT - + >>> from systems.tests.testdata import get_test_object_futures_with_pos_sizing >>> from systems.basesystem import System >>> (posobject, combobject, capobject, rules, rawdata, data, config)=get_test_object_futures_with_pos_sizing() >>> system=System([rawdata, rules, posobject, combobject, capobject,PortfoliosFixed()], data, config) - >>> + >>> >>> ## from config >>> system.portfolio.get_subsystem_position("EDOLLAR").tail(2) ss_position @@ -62,25 +63,24 @@ def get_subsystem_position(self, instrument_code): """ return self.parent.positionSize.get_subsystem_position(instrument_code) - def get_raw_instrument_weights(self): """ Get the instrument weights - + These are 'raw' because we need to account for potentially missing positions, and weights that don't add up. - + From: (a) passed into subsystem when created (b) ... if not found then: in system.config.instrument_weights - - :returns: TxK pd.DataFrame containing weights, columns are instrument names, T covers all subsystem positions + + :returns: TxK pd.DataFrame containing weights, columns are instrument names, T covers all subsystem positions >>> from systems.tests.testdata import get_test_object_futures_with_pos_sizing >>> from systems.basesystem import System >>> (posobject, combobject, capobject, rules, rawdata, data, config)=get_test_object_futures_with_pos_sizing() >>> config.instrument_weights=dict(EDOLLAR=0.1, US10=0.9) >>> system=System([rawdata, rules, posobject, combobject, capobject,PortfoliosFixed()], data, config) - >>> + >>> >>> ## from config >>> system.portfolio.get_instrument_weights().tail(2) EDOLLAR US10 @@ -94,85 +94,90 @@ def get_raw_instrument_weights(self): EDOLLAR US10 2015-04-21 0.5 0.5 2015-04-22 0.5 0.5 - """ - def _get_instrument_weights(system, an_ignored_variable, this_stage ): + """ + def _get_instrument_weights(system, an_ignored_variable, this_stage): try: - instrument_weights=system.config.instrument_weights + instrument_weights = system.config.instrument_weights except: - instruments=self.parent.data.get_instrument_list() - weight=1.0/len(instruments) - - print("WARNING: No instrument weights - using equal weights of %.4f over all %d instruments in data" % (weight, len(instruments))) - instrument_weights=dict([(instrument_code, weight) for instrument_code in instruments]) - - ## Now we have a dict, fixed_weights. - ## Need to turn into a timeseries covering the range of forecast dates - instrument_list=list(instrument_weights.keys()) - instrument_list.sort() - - subsys_ts=[ - this_stage.get_subsystem_position(instrument_code).index - for instrument_code in instrument_list] - - earliest_date=min([min(fts) for fts in subsys_ts]) - latest_date=max([max(fts) for fts in subsys_ts]) - - ## this will be daily, but will be resampled later - weight_ts=pd.date_range(start=earliest_date, end=latest_date) - - instrument_weights_weights=dict([ - (instrument_code, pd.Series([instrument_weights[instrument_code]]*len(weight_ts), index=weight_ts)) - for instrument_code in instrument_list]) - - instrument_weights_weights=pd.concat(instrument_weights_weights, axis=1) - instrument_weights_weights.columns=instrument_list + instruments = self.parent.data.get_instrument_list() + weight = 1.0 / len(instruments) - return instrument_weights_weights - - instrument_weights=self.parent.calc_or_cache( "get_raw_instrument_weights", ALL_KEYNAME, _get_instrument_weights, self) - return instrument_weights + print("WARNING: No instrument weights - using equal weights of %.4f over all %d instruments in data" % + (weight, len(instruments))) + instrument_weights = dict( + [(instrument_code, weight) for instrument_code in instruments]) + + # Now we have a dict, fixed_weights. + # Need to turn into a timeseries covering the range of forecast + # dates + instrument_list = sorted(instrument_weights.keys()) + + subsys_ts = [ + this_stage.get_subsystem_position(instrument_code).index + for instrument_code in instrument_list] + + earliest_date = min([min(fts) for fts in subsys_ts]) + latest_date = max([max(fts) for fts in subsys_ts]) + # this will be daily, but will be resampled later + weight_ts = pd.date_range(start=earliest_date, end=latest_date) + instrument_weights_weights = dict([ + (instrument_code, pd.Series([instrument_weights[ + instrument_code]] * len(weight_ts), index=weight_ts)) + for instrument_code in instrument_list]) + + instrument_weights_weights = pd.concat( + instrument_weights_weights, axis=1) + instrument_weights_weights.columns = instrument_list + + return instrument_weights_weights + + instrument_weights = self.parent.calc_or_cache( + "get_raw_instrument_weights", ALL_KEYNAME, _get_instrument_weights, self) + return instrument_weights def get_instrument_weights(self): """ Get the time series of instrument weights, accounting for potentially missing positions, and weights that don't add up. - - :returns: TxK pd.DataFrame containing weights, columns are instrument names, T covers all subsystem positions + :returns: TxK pd.DataFrame containing weights, columns are instrument names, T covers all subsystem positions + + + """ + def _get_clean_instrument_weights( + system, an_ignored_variable, this_stage): + + raw_instr_weights = this_stage.get_raw_instrument_weights() + instrument_list = list(raw_instr_weights.columns) + + subsys_positions = [this_stage.get_subsystem_position(instrument_code) + for instrument_code in instrument_list] - """ - def _get_clean_instrument_weights(system, an_ignored_variable, this_stage ): + subsys_positions = pd.concat(subsys_positions, axis=1).ffill() + subsys_positions.columns = instrument_list - raw_instr_weights=this_stage.get_raw_instrument_weights() - instrument_list=list(raw_instr_weights.columns) - - subsys_positions=[this_stage.get_subsystem_position(instrument_code) - for instrument_code in instrument_list] - - subsys_positions=pd.concat(subsys_positions, axis=1).ffill() - subsys_positions.columns=instrument_list - - instrument_weights=fix_weights_vs_pdm(raw_instr_weights, subsys_positions) + instrument_weights = fix_weights_vs_pdm( + raw_instr_weights, subsys_positions) return instrument_weights - - instrument_weights=self.parent.calc_or_cache( "get_instrument_weights", ALL_KEYNAME, _get_clean_instrument_weights, self) + + instrument_weights = self.parent.calc_or_cache( + "get_instrument_weights", ALL_KEYNAME, _get_clean_instrument_weights, self) return instrument_weights - def get_instrument_diversification_multiplier(self): """ Get the instrument diversification multiplier - - :returns: TxK pd.DataFrame containing weights, columns are instrument names, T covers all subsystem positions + + :returns: TxK pd.DataFrame containing weights, columns are instrument names, T covers all subsystem positions >>> from systems.tests.testdata import get_test_object_futures_with_pos_sizing >>> from systems.basesystem import System >>> (posobject, combobject, capobject, rules, rawdata, data, config)=get_test_object_futures_with_pos_sizing() >>> system=System([rawdata, rules, posobject, combobject, capobject,PortfoliosFixed()], data, config) - >>> + >>> >>> ## from config >>> system.portfolio.get_instrument_diversification_multiplier().tail(2) idm @@ -186,49 +191,52 @@ def get_instrument_diversification_multiplier(self): idm 2015-04-21 1 2015-04-22 1 - """ - def _get_instrument_div_multiplier(system, an_ignored_variable, this_stage ): + """ + def _get_instrument_div_multiplier( + system, an_ignored_variable, this_stage): if hasattr(system.config, "instrument_div_multiplier"): - div_mult=system.config.instrument_div_multiplier + div_mult = system.config.instrument_div_multiplier elif "instrument_div_multiplier" in system_defaults: - div_mult=system_defaults["instrument_div_multiplier"] + div_mult = system_defaults["instrument_div_multiplier"] else: - raise Exception("Instrument div. multiplier must be in system.config or system_defaults") - - ## Now we have a fixed weight - ## Need to turn into a timeseries covering the range of forecast dates + raise Exception( + "Instrument div. multiplier must be in system.config or system_defaults") - ## this will be daily, but will be resampled later - weight_ts=this_stage.get_instrument_weights().index - - ts_idm=pd.Series([div_mult]*len(weight_ts), index=weight_ts).to_frame("idm") + # Now we have a fixed weight + # Need to turn into a timeseries covering the range of forecast + # dates - return ts_idm - - instrument_div_multiplier=self.parent.calc_or_cache( "get_instrument_diversification_multiplier", ALL_KEYNAME, _get_instrument_div_multiplier, self) - return instrument_div_multiplier + # this will be daily, but will be resampled later + weight_ts = this_stage.get_instrument_weights().index + ts_idm = pd.Series([div_mult] * len(weight_ts), + index=weight_ts).to_frame("idm") + return ts_idm + + instrument_div_multiplier = self.parent.calc_or_cache( + "get_instrument_diversification_multiplier", ALL_KEYNAME, _get_instrument_div_multiplier, self) + return instrument_div_multiplier def get_notional_position(self, instrument_code): """ Gets the position, accounts for instrument weights and diversification multiplier - + Note: At this stage we're dealing with a notional, fixed, amount of capital. We'll need to work out p&l to scale positions properly - + :param instrument_code: instrument to get values for :type instrument_code: str - - :returns: Tx1 pd.DataFrame - + + :returns: Tx1 pd.DataFrame + KEY OUTPUT >>> from systems.tests.testdata import get_test_object_futures_with_pos_sizing >>> from systems.basesystem import System >>> (posobject, combobject, capobject, rules, rawdata, data, config)=get_test_object_futures_with_pos_sizing() >>> system=System([rawdata, rules, posobject, combobject, capobject,PortfoliosFixed()], data, config) - >>> + >>> >>> ## from config >>> system.portfolio.get_notional_position("EDOLLAR").tail(2) pos @@ -236,27 +244,30 @@ def get_notional_position(self, instrument_code): 2015-04-22 4.125137 >>> - """ - def _get_notional_position(system, instrument_code, this_stage ): - idm=this_stage.get_instrument_diversification_multiplier() - instr_weights=this_stage.get_instrument_weights() - subsys_position=this_stage.get_subsystem_position(instrument_code) - - inst_weight_this_code=instr_weights[instrument_code].to_frame("weight") - - inst_weight_this_code=inst_weight_this_code.reindex(subsys_position.index).ffill() - idm=idm.reindex(subsys_position.index).ffill() - - multiplier=multiply_df_single_column(inst_weight_this_code, idm) - notional_position=multiply_df_single_column(subsys_position, multiplier) - notional_position.columns=['pos'] + """ + def _get_notional_position(system, instrument_code, this_stage): + idm = this_stage.get_instrument_diversification_multiplier() + instr_weights = this_stage.get_instrument_weights() + subsys_position = this_stage.get_subsystem_position( + instrument_code) + + inst_weight_this_code = instr_weights[ + instrument_code].to_frame("weight") + + inst_weight_this_code = inst_weight_this_code.reindex( + subsys_position.index).ffill() + idm = idm.reindex(subsys_position.index).ffill() + + multiplier = multiply_df_single_column(inst_weight_this_code, idm) + notional_position = multiply_df_single_column( + subsys_position, multiplier) + notional_position.columns = ['pos'] return notional_position - - notional_position=self.parent.calc_or_cache( "get_notional_position", instrument_code, _get_notional_position, self) - return notional_position - + notional_position = self.parent.calc_or_cache( + "get_notional_position", instrument_code, _get_notional_position, self) + return notional_position if __name__ == '__main__': diff --git a/systems/positionsizing.py b/systems/positionsizing.py index d5646f2119..7838dee76c 100644 --- a/systems/positionsizing.py +++ b/systems/positionsizing.py @@ -5,62 +5,63 @@ from syscore.dateutils import ROOT_BDAYS_INYEAR from syscore.algos import robust_vol_calc + class PositionSizing(SystemStage): """ Stage for position sizing (take combined forecast; turn into subsystem positions) - + KEY INPUTS: a) system.combForecast.get_combined_forecast(instrument_code) found in self.get_combined_forecast - + b) system.rawdata.get_daily_percentage_volatility(instrument_code) found in self.get_price_volatility(instrument_code) - + If not found, uses system.data.get_instrument_price to calculate - + c) system.rawdata.daily_denominator_price((instrument_code) found in self.get_instrument_sizing_data(instrument_code) - If not found, uses system.data.get_instrument_price + If not found, uses system.data.get_instrument_price d) system.data.get_value_of_block_price_move(instrument_code) found in self.get_instrument_sizing_data(instrument_code) - + e) system.data.get_fx_for_instrument(instrument_code, base_currency) found in self.get_fx_rate(instrument_code) - - + + KEY OUTPUT: system.positionSize.get_subsystem_position(instrument_code) Name: positionSize """ - + def __init__(self): """ Create a SystemStage for combining forecasts - - + + """ - protected=['get_daily_cash_vol_target'] + protected = ['get_daily_cash_vol_target'] setattr(self, "_protected", protected) setattr(self, "name", "positionSize") - + def get_combined_forecast(self, instrument_code): """ Get the combined forecast from previous module - + :param instrument_code: instrument to get values for :type instrument_code: str - - :returns: Tx1 pd.DataFrame - + + :returns: Tx1 pd.DataFrame + KEY INPUT - + >>> from systems.tests.testdata import get_test_object_futures_with_comb_forecasts >>> from systems.basesystem import System >>> (comb, fcs, rules, rawdata, data, config)=get_test_object_futures_with_comb_forecasts() >>> system=System([rawdata, rules, fcs, comb, PositionSizing()], data, config) - >>> + >>> >>> system.positionSize.get_combined_forecast("EDOLLAR").tail(2) comb_forecast 2015-04-21 7.622781 @@ -69,65 +70,63 @@ def get_combined_forecast(self, instrument_code): """ return self.parent.combForecast.get_combined_forecast(instrument_code) - + def get_price_volatility(self, instrument_code): """ - Get the daily % volatility - + Get the daily % volatility + :param instrument_code: instrument to get values for :type instrument_code: str - - :returns: Tx1 pd.DataFrame - + + :returns: Tx1 pd.DataFrame + KEY INPUT >>> from systems.tests.testdata import get_test_object_futures_with_comb_forecasts >>> from systems.basesystem import System >>> (comb, fcs, rules, rawdata, data, config)=get_test_object_futures_with_comb_forecasts() >>> system=System([rawdata, rules, fcs, comb, PositionSizing()], data, config) - >>> + >>> >>> system.positionSize.get_price_volatility("EDOLLAR").tail(2) vol 2015-04-21 0.058307 2015-04-22 0.059634 >>> >>> system2=System([ rules, fcs, comb, PositionSizing()], data, config) - >>> + >>> >>> system2.positionSize.get_price_volatility("EDOLLAR").tail(2) vol 2015-04-21 0.058274 2015-04-22 0.059632 """ if hasattr(self.parent, "rawdata"): - daily_perc_vol=self.parent.rawdata.get_daily_percentage_volatility(instrument_code) + daily_perc_vol = self.parent.rawdata.get_daily_percentage_volatility( + instrument_code) else: - price=self.parent.data.get_instrument_price(instrument_code) - price=price.resample("1B", how="last") - return_vol=robust_vol_calc(price.diff()) - daily_perc_vol=100.0*divide_df_single_column(return_vol,price) - - return daily_perc_vol - - + price = self.parent.data.get_instrument_price(instrument_code) + price = price.resample("1B", how="last") + return_vol = robust_vol_calc(price.diff()) + daily_perc_vol = 100.0 * divide_df_single_column(return_vol, price) + return daily_perc_vol def get_instrument_sizing_data(self, instrument_code): """ Get various things from data and rawdata to calculate position sizes - + KEY INPUT - + :param instrument_code: instrument to get values for :type instrument_code: str - - :returns: tuple (Tx1 pd.DataFrame: underlying price [as used to work out % volatility], - float: value of price block move) + + :returns: tuple (Tx1 pd.DataFrame: underlying price [as used to work out % volatility], + float: value of price block move) >>> from systems.tests.testdata import get_test_object_futures_with_comb_forecasts >>> from systems.basesystem import System >>> (comb, fcs, rules, rawdata, data, config)=get_test_object_futures_with_comb_forecasts() >>> system=System([rawdata, rules, fcs, comb, PositionSizing()], data, config) - >>> + >>> >>> ans=system.positionSize.get_instrument_sizing_data("EDOLLAR") >>> ans[0].tail(2) price @@ -138,7 +137,7 @@ def get_instrument_sizing_data(self, instrument_code): 2500 >>> >>> system=System([rules, fcs, comb, PositionSizing()], data, config) - >>> + >>> >>> ans=system.positionSize.get_instrument_sizing_data("EDOLLAR") >>> ans[0].tail(2) price @@ -152,36 +151,37 @@ def get_instrument_sizing_data(self, instrument_code): """ if hasattr(self.parent, "rawdata"): - underlying_price=self.parent.rawdata.daily_denominator_price(instrument_code) + underlying_price = self.parent.rawdata.daily_denominator_price( + instrument_code) else: - underlying_price=self.parent.data.get_instrument_price(instrument_code) - underlying_price=underlying_price.resample("1B", how="last") - - value_of_price_move=self.parent.data.get_value_of_block_price_move(instrument_code) - - return (underlying_price, value_of_price_move) + underlying_price = self.parent.data.get_instrument_price( + instrument_code) + underlying_price = underlying_price.resample("1B", how="last") + value_of_price_move = self.parent.data.get_value_of_block_price_move( + instrument_code) + return (underlying_price, value_of_price_move) def get_daily_cash_vol_target(self): """ Get the daily cash vol target - + Requires: percentage_vol_target, notional_trading_capital, base_currency - + To find these, look in (a) in system.config.parameters... (b).... if not found, in systems.get_defaults.py - - :Returns: tuple (str, float): str is base_currency, float is value + + :Returns: tuple (str, float): str is base_currency, float is value >>> from systems.tests.testdata import get_test_object_futures_with_comb_forecasts >>> from systems.basesystem import System >>> (comb, fcs, rules, rawdata, data, config)=get_test_object_futures_with_comb_forecasts() >>> system=System([rawdata, rules, fcs, comb, PositionSizing()], data, config) >>> - >>> ## from config + >>> ## from config >>> system.positionSize.get_daily_cash_vol_target()['base_currency'] 'GBP' >>> @@ -191,86 +191,90 @@ def get_daily_cash_vol_target(self): >>> system.positionSize.get_daily_cash_vol_target()['base_currency'] 'USD' >>> - + """ - def _get_vol_target(system, an_ignored_variable, this_stage ): + def _get_vol_target(system, an_ignored_variable, this_stage): try: - percentage_vol_target=system.config.percentage_vol_target + percentage_vol_target = system.config.percentage_vol_target except: - percentage_vol_target=system_defaults['percentage_vol_target'] - + percentage_vol_target = system_defaults[ + 'percentage_vol_target'] + try: - notional_trading_capital=system.config.notional_trading_capital + notional_trading_capital = system.config.notional_trading_capital except: - notional_trading_capital=system_defaults['notional_trading_capital'] - + notional_trading_capital = system_defaults[ + 'notional_trading_capital'] + try: - base_currency=system.config.base_currency + base_currency = system.config.base_currency except: - base_currency=system_defaults['base_currency'] - - annual_cash_vol_target=notional_trading_capital*percentage_vol_target/100.0 - daily_cash_vol_target=annual_cash_vol_target/ROOT_BDAYS_INYEAR - - vol_target_dict=dict(base_currency=base_currency, percentage_vol_target=percentage_vol_target, - notional_trading_capital=notional_trading_capital, annual_cash_vol_target=annual_cash_vol_target, - daily_cash_vol_target=daily_cash_vol_target) - + base_currency = system_defaults['base_currency'] + + annual_cash_vol_target = notional_trading_capital * percentage_vol_target / 100.0 + daily_cash_vol_target = annual_cash_vol_target / ROOT_BDAYS_INYEAR + + vol_target_dict = dict(base_currency=base_currency, percentage_vol_target=percentage_vol_target, + notional_trading_capital=notional_trading_capital, annual_cash_vol_target=annual_cash_vol_target, + daily_cash_vol_target=daily_cash_vol_target) + return vol_target_dict - - vol_target_dict=self.parent.calc_or_cache( 'get_daily_cash_vol_target', ALL_KEYNAME, _get_vol_target, self) + + vol_target_dict = self.parent.calc_or_cache( + 'get_daily_cash_vol_target', ALL_KEYNAME, _get_vol_target, self) return vol_target_dict - def get_fx_rate(self, instrument_code): """ Get FX rate to translate instrument volatility into same currency as account value. - + KEY INPUT - + :param instrument_code: instrument to get values for :type instrument_code: str - + :returns: Tx1 pd.DataFrame: fx rate >>> from systems.tests.testdata import get_test_object_futures_with_comb_forecasts >>> from systems.basesystem import System >>> (comb, fcs, rules, rawdata, data, config)=get_test_object_futures_with_comb_forecasts() >>> system=System([rawdata, rules, fcs, comb, PositionSizing()], data, config) - >>> + >>> >>> system.positionSize.get_fx_rate("EDOLLAR").tail(2) fx 2015-10-30 0.654270 2015-11-02 0.650542 """ - def _get_fx_rate(system, instrument_code, this_stage ): - base_currency=this_stage.get_daily_cash_vol_target()['base_currency'] - fx_rate=system.data.get_fx_for_instrument(instrument_code, base_currency) - + def _get_fx_rate(system, instrument_code, this_stage): + base_currency = this_stage.get_daily_cash_vol_target()[ + 'base_currency'] + fx_rate = system.data.get_fx_for_instrument( + instrument_code, base_currency) + return fx_rate - - fx_rate=self.parent.calc_or_cache( 'get_fx_rate', instrument_code, _get_fx_rate, self) - + + fx_rate = self.parent.calc_or_cache( + 'get_fx_rate', instrument_code, _get_fx_rate, self) + return fx_rate - - + def get_block_value(self, instrument_code): """ Calculate block value for instrument_code :param instrument_code: instrument to get values for :type instrument_code: str - - :returns: Tx1 pd.DataFrame + + :returns: Tx1 pd.DataFrame >>> from systems.tests.testdata import get_test_object_futures_with_comb_forecasts >>> from systems.basesystem import System >>> (comb, fcs, rules, rawdata, data, config)=get_test_object_futures_with_comb_forecasts() >>> system=System([rawdata, rules, fcs, comb, PositionSizing()], data, config) - >>> + >>> >>> system.positionSize.get_block_value("EDOLLAR").tail(2) price 2015-04-21 2445.75 @@ -282,32 +286,33 @@ def get_block_value(self, instrument_code): 2015-04-21 2447.6250 2015-04-22 2445.8125 - """ - def _get_block_value(system, instrument_code, this_stage ): - - (underlying_price, value_of_price_move)=this_stage.get_instrument_sizing_data(instrument_code) - block_value=0.01*underlying_price*value_of_price_move - + """ + def _get_block_value(system, instrument_code, this_stage): + + (underlying_price, value_of_price_move) = this_stage.get_instrument_sizing_data( + instrument_code) + block_value = 0.01 * underlying_price * value_of_price_move + return block_value - - block_value=self.parent.calc_or_cache( 'get_block_value', instrument_code, _get_block_value, self) - return block_value + block_value = self.parent.calc_or_cache( + 'get_block_value', instrument_code, _get_block_value, self) + return block_value def get_instrument_currency_vol(self, instrument_code): """ - Get value of volatility of instrument in instrument's own currency - + Get value of volatility of instrument in instrument's own currency + :param instrument_code: instrument to get values for :type instrument_code: str - - :returns: Tx1 pd.DataFrame + + :returns: Tx1 pd.DataFrame >>> from systems.tests.testdata import get_test_object_futures_with_comb_forecasts >>> from systems.basesystem import System >>> (comb, fcs, rules, rawdata, data, config)=get_test_object_futures_with_comb_forecasts() >>> system=System([rawdata, rules, fcs, comb, PositionSizing()], data, config) - >>> + >>> >>> system.positionSize.get_instrument_currency_vol("EDOLLAR").tail(2) icv 2015-04-21 142.603997 @@ -318,83 +323,84 @@ def get_instrument_currency_vol(self, instrument_code): icv 2015-04-21 142.633151 2015-04-22 145.849773 - + """ - def _get_instrument_currency_vol(system, instrument_code, this_stage ): - - block_value=this_stage.get_block_value(instrument_code) - daily_perc_vol=this_stage.get_price_volatility(instrument_code) - - instr_ccy_vol=multiply_df_single_column(block_value, daily_perc_vol, ffill=(True, False)) - instr_ccy_vol.columns=['icv'] - + def _get_instrument_currency_vol(system, instrument_code, this_stage): + + block_value = this_stage.get_block_value(instrument_code) + daily_perc_vol = this_stage.get_price_volatility(instrument_code) + + instr_ccy_vol = multiply_df_single_column( + block_value, daily_perc_vol, ffill=(True, False)) + instr_ccy_vol.columns = ['icv'] + return instr_ccy_vol - - instr_ccy_vol=self.parent.calc_or_cache( 'get_instrument_currency_vol', instrument_code, _get_instrument_currency_vol, self) + instr_ccy_vol = self.parent.calc_or_cache( + 'get_instrument_currency_vol', instrument_code, _get_instrument_currency_vol, self) return instr_ccy_vol def get_instrument_value_vol(self, instrument_code): """ - Get value of volatility of instrument in base currency (used for account value) - + Get value of volatility of instrument in base currency (used for account value) + :param instrument_code: instrument to get values for :type instrument_code: str - - :returns: Tx1 pd.DataFrame + + :returns: Tx1 pd.DataFrame >>> from systems.tests.testdata import get_test_object_futures_with_comb_forecasts >>> from systems.basesystem import System >>> (comb, fcs, rules, rawdata, data, config)=get_test_object_futures_with_comb_forecasts() >>> system=System([rawdata, rules, fcs, comb, PositionSizing()], data, config) - >>> + >>> >>> system.positionSize.get_instrument_value_vol("EDOLLAR").tail(2) ivv 2015-04-21 95.408349 2015-04-22 97.782721 - >>> + >>> >>> system2=System([rawdata, rules, fcs, comb, PositionSizing()], data, config) >>> system2.positionSize.get_instrument_value_vol("EDOLLAR").tail(2) ivv 2015-04-21 95.408349 2015-04-22 97.782721 - + """ - def _get_instrument_value_vol(system, instrument_code, this_stage ): - - instr_ccy_vol=this_stage.get_instrument_currency_vol(instrument_code) - fx_rate=this_stage.get_fx_rate(instrument_code) - - instr_value_vol=multiply_df_single_column(instr_ccy_vol, fx_rate, ffill=(False, True)) - instr_value_vol.columns=['ivv'] - + def _get_instrument_value_vol(system, instrument_code, this_stage): + + instr_ccy_vol = this_stage.get_instrument_currency_vol( + instrument_code) + fx_rate = this_stage.get_fx_rate(instrument_code) + + instr_value_vol = multiply_df_single_column( + instr_ccy_vol, fx_rate, ffill=(False, True)) + instr_value_vol.columns = ['ivv'] + return instr_value_vol - - instr_value_vol=self.parent.calc_or_cache( 'get_instrument_value_vol', instrument_code, _get_instrument_value_vol, self) + instr_value_vol = self.parent.calc_or_cache( + 'get_instrument_value_vol', instrument_code, _get_instrument_value_vol, self) return instr_value_vol - - def get_volatility_scalar(self, instrument_code): """ - Get ratio of required volatility vs volatility of instrument in instrument's own currency - + Get ratio of required volatility vs volatility of instrument in instrument's own currency + :param instrument_code: instrument to get values for :type instrument_code: str - - :returns: Tx1 pd.DataFrame + + :returns: Tx1 pd.DataFrame >>> from systems.tests.testdata import get_test_object_futures_with_comb_forecasts >>> from systems.basesystem import System >>> (comb, fcs, rules, rawdata, data, config)=get_test_object_futures_with_comb_forecasts() >>> system=System([rawdata, rules, fcs, comb, PositionSizing()], data, config) - >>> + >>> >>> system.positionSize.get_volatility_scalar("EDOLLAR").tail(2) vol_scalar 2015-04-21 10.481263 2015-04-22 10.226755 - >>> + >>> >>> ## without raw data >>> system2=System([ rules, fcs, comb, PositionSizing()], data, config) >>> system2.positionSize.get_volatility_scalar("EDOLLAR").tail(2) @@ -402,41 +408,43 @@ def get_volatility_scalar(self, instrument_code): 2015-04-21 10.479121 2015-04-22 10.226756 """ - def _get_volatility_scalar(system, instrument_code, this_stage ): - - instr_value_vol=this_stage.get_instrument_value_vol(instrument_code) - cash_vol_target=this_stage.get_daily_cash_vol_target()['daily_cash_vol_target'] - - vol_scalar=cash_vol_target/ instr_value_vol - vol_scalar.columns=['vol_scalar'] - + def _get_volatility_scalar(system, instrument_code, this_stage): + + instr_value_vol = this_stage.get_instrument_value_vol( + instrument_code) + cash_vol_target = this_stage.get_daily_cash_vol_target()[ + 'daily_cash_vol_target'] + + vol_scalar = cash_vol_target / instr_value_vol + vol_scalar.columns = ['vol_scalar'] + return vol_scalar - vol_scalar=self.parent.calc_or_cache( 'get_volatility_scalar', instrument_code, _get_volatility_scalar, self) + vol_scalar = self.parent.calc_or_cache( + 'get_volatility_scalar', instrument_code, _get_volatility_scalar, self) return vol_scalar - def get_subsystem_position(self, instrument_code): """ - Get scaled position (assuming for now we trade our entire capital for one instrument) + Get scaled position (assuming for now we trade our entire capital for one instrument) + + KEY OUTPUT - KEY OUTPUT - :param instrument_code: instrument to get values for :type instrument_code: str - - :returns: Tx1 pd.DataFrame + + :returns: Tx1 pd.DataFrame >>> from systems.tests.testdata import get_test_object_futures_with_comb_forecasts >>> from systems.basesystem import System >>> (comb, fcs, rules, rawdata, data, config)=get_test_object_futures_with_comb_forecasts() >>> system=System([rawdata, rules, fcs, comb, PositionSizing()], data, config) - >>> + >>> >>> system.positionSize.get_subsystem_position("EDOLLAR").tail(2) ss_position 2015-04-21 7.989637 2015-04-22 6.875228 - >>> + >>> >>> system2=System([rawdata, rules, fcs, comb, PositionSizing()], data, config) >>> system2.positionSize.get_subsystem_position("EDOLLAR").tail(2) ss_position @@ -444,22 +452,23 @@ def get_subsystem_position(self, instrument_code): 2015-04-22 6.875228 """ - def _get_subsystem_position(system, instrument_code, this_stage ): + def _get_subsystem_position(system, instrument_code, this_stage): """ We don't allow this to be changed in config """ - avg_abs_forecast=system_defaults['average_absolute_forecast'] - - vol_scalar=this_stage.get_volatility_scalar(instrument_code) - forecast=this_stage.get_combined_forecast(instrument_code) - - subsystem_position=multiply_df_single_column(vol_scalar, forecast, ffill=(True, False))/avg_abs_forecast - subsystem_position.columns=['ss_position'] - + avg_abs_forecast = system_defaults['average_absolute_forecast'] + + vol_scalar = this_stage.get_volatility_scalar(instrument_code) + forecast = this_stage.get_combined_forecast(instrument_code) + + subsystem_position = multiply_df_single_column( + vol_scalar, forecast, ffill=(True, False)) / avg_abs_forecast + subsystem_position.columns = ['ss_position'] + return subsystem_position - - subsystem_position=self.parent.calc_or_cache( 'get_subsystem_position', instrument_code, _get_subsystem_position, self) + subsystem_position = self.parent.calc_or_cache( + 'get_subsystem_position', instrument_code, _get_subsystem_position, self) return subsystem_position if __name__ == '__main__': diff --git a/systems/provided/example/rules.py b/systems/provided/example/rules.py index b18d8f544d..36eda85a3f 100644 --- a/systems/provided/example/rules.py +++ b/systems/provided/example/rules.py @@ -5,67 +5,72 @@ from syscore.algos import robust_vol_calc from syscore.pdutils import divide_df_single_column + def ewmac_forecast_with_defaults(price, Lfast=32, Lslow=128): """ Calculate the ewmac trading fule forecast, given a price and EWMA speeds Lfast, Lslow and vol_lookback - + Assumes that 'price' is daily data This version recalculates the price volatility, and does not do capping or scaling :param price: The price or other series to use (assumed Tx1) :type price: pd.DataFrame - :param Lfast: Lookback for fast in days + :param Lfast: Lookback for fast in days :type Lfast: int :param Lslow: Lookback for slow in days - :type Lslow: int - + :type Lslow: int + :returns: pd.DataFrame -- unscaled, uncapped forecast - + """ - ## price: This is the stitched price series - ## We can't use the price of the contract we're trading, or the volatility will be jumpy - ## And we'll miss out on the rolldown. See http://qoppac.blogspot.co.uk/2015/05/systems-building-futures-rolling.html - - ## We don't need to calculate the decay parameter, just use the span directly - - fast_ewma=pd.ewma(price, span=Lfast) - slow_ewma=pd.ewma(price, span=Lslow) - raw_ewmac=fast_ewma - slow_ewma - - vol=robust_vol_calc(price.diff()) - - return divide_df_single_column(raw_ewmac,vol) + # price: This is the stitched price series + # We can't use the price of the contract we're trading, or the volatility will be jumpy + # And we'll miss out on the rolldown. See + # http://qoppac.blogspot.co.uk/2015/05/systems-building-futures-rolling.html + + # We don't need to calculate the decay parameter, just use the span + # directly + + fast_ewma = pd.ewma(price, span=Lfast) + slow_ewma = pd.ewma(price, span=Lslow) + raw_ewmac = fast_ewma - slow_ewma + + vol = robust_vol_calc(price.diff()) + + return divide_df_single_column(raw_ewmac, vol) + def ewmac_forecast_with_defaults_no_vol(price, vol, Lfast=16, Lslow=32): """ Calculate the ewmac trading fule forecast, given a price and EWMA speeds Lfast, Lslow and vol_lookback - + Assumes that 'price' is daily data This version recalculates the price volatility, and does not do capping or scaling :param price: The price or other series to use (assumed Tx1) :type price: pd.DataFrame - :param Lfast: Lookback for fast in days + :param Lfast: Lookback for fast in days :type Lfast: int :param Lslow: Lookback for slow in days - :type Lslow: int - + :type Lslow: int + :returns: pd.DataFrame -- unscaled, uncapped forecast - + """ - ## price: This is the stitched price series - ## We can't use the price of the contract we're trading, or the volatility will be jumpy - ## And we'll miss out on the rolldown. See http://qoppac.blogspot.co.uk/2015/05/systems-building-futures-rolling.html - - ## We don't need to calculate the decay parameter, just use the span directly - - fast_ewma=pd.ewma(price, span=Lfast) - slow_ewma=pd.ewma(price, span=Lslow) - raw_ewmac=fast_ewma - slow_ewma - - return divide_df_single_column(raw_ewmac,vol) + # price: This is the stitched price series + # We can't use the price of the contract we're trading, or the volatility will be jumpy + # And we'll miss out on the rolldown. See + # http://qoppac.blogspot.co.uk/2015/05/systems-building-futures-rolling.html + + # We don't need to calculate the decay parameter, just use the span + # directly + + fast_ewma = pd.ewma(price, span=Lfast) + slow_ewma = pd.ewma(price, span=Lslow) + raw_ewmac = fast_ewma - slow_ewma + return divide_df_single_column(raw_ewmac, vol) diff --git a/systems/provided/example/simplesystem.py b/systems/provided/example/simplesystem.py index d5cfa45e81..c4eca1ebf3 100644 --- a/systems/provided/example/simplesystem.py +++ b/systems/provided/example/simplesystem.py @@ -15,13 +15,11 @@ def simplesystem(data=None, config=None): Example of how to 'wrap' a complete system """ if config is None: - config=Config("systems.provided.example.simplesystemconfig.yaml") + config = Config("systems.provided.example.simplesystemconfig.yaml") if data is None: - data=csvFuturesData() + data = csvFuturesData() - my_system=System([Account(), PortfoliosFixed(), PositionSizing(), ForecastCombineFixed(), ForecastScaleCapFixed(), Rules() - ], data, config) + my_system = System([Account(), PortfoliosFixed(), PositionSizing(), ForecastCombineFixed(), ForecastScaleCapFixed(), Rules() + ], data, config) return my_system - - diff --git a/systems/provided/futures_chapter15/basesystem.py b/systems/provided/futures_chapter15/basesystem.py index 1bd5523ab2..e74e3338a6 100644 --- a/systems/provided/futures_chapter15/basesystem.py +++ b/systems/provided/futures_chapter15/basesystem.py @@ -17,18 +17,18 @@ from systems.account import Account -def futures_system( data=None, config=None, trading_rules=None): +def futures_system(data=None, config=None, trading_rules=None): """ - + :param data: data object (defaults to reading from csv files) :type data: sysdata.data.Data, or anything that inherits from it - + :param config: Configuration object (defaults to futuresconfig.yaml in this directory) :type config: sysdata.configdata.Config - + :param trading_rules: Set of trading rules to use (defaults to set specified in config object) :param trading_rules: list or dict of TradingRules, or something that can be parsed to that - + >>> system=futures_system() >>> system System with stages: accounts, portfolio, positionSize, rawdata, combForecast, forecastScaleCap, rules @@ -41,21 +41,22 @@ def futures_system( data=None, config=None, trading_rules=None): 2015-04-21 0.350892 2015-04-22 0.350892 """ - + if data is None: - data=csvFuturesData() - + data = csvFuturesData() + if config is None: - config=Config("systems.provided.futures_chapter15.futuresconfig.yaml") - - rules=Rules(trading_rules) + config = Config( + "systems.provided.futures_chapter15.futuresconfig.yaml") + + rules = Rules(trading_rules) + + system = System([Account(), PortfoliosFixed(), PositionSizing(), FuturesRawData(), ForecastCombineFixed(), + ForecastScaleCapFixed(), rules], data, config) - system=System([Account(), PortfoliosFixed(), PositionSizing(), FuturesRawData(), ForecastCombineFixed(), - ForecastScaleCapFixed(), rules], data, config) - return system - - + + if __name__ == '__main__': import doctest - doctest.testmod() \ No newline at end of file + doctest.testmod() diff --git a/systems/provided/futures_chapter15/rules.py b/systems/provided/futures_chapter15/rules.py index aaacffabe6..a9c20b9cb9 100644 --- a/systems/provided/futures_chapter15/rules.py +++ b/systems/provided/futures_chapter15/rules.py @@ -5,10 +5,11 @@ from syscore.pdutils import divide_df_single_column import pandas as pd + def ewmac(price, vol, Lfast, Lslow): """ Calculate the ewmac trading fule forecast, given a price and EWMA speeds Lfast, Lslow and vol_lookback - + Assumes that 'price' is daily data This version recalculates the price volatility, and does not do capping or scaling @@ -16,17 +17,17 @@ def ewmac(price, vol, Lfast, Lslow): :param price: The price or other series to use (assumed Tx1) :type price: pd.DataFrame - :param vol: The daily price unit volatility (NOT % vol) + :param vol: The daily price unit volatility (NOT % vol) :type vol: pd.DataFrame (assumed Tx1) - :param Lfast: Lookback for fast in days + :param Lfast: Lookback for fast in days :type Lfast: int :param Lslow: Lookback for slow in days - :type Lslow: int - + :type Lslow: int + :returns: pd.DataFrame -- unscaled, uncapped forecast - + >>> from systems.provided.example.testdata import get_test_object_futures >>> from systems.basesystem import System @@ -38,28 +39,31 @@ def ewmac(price, vol, Lfast, Lslow): 2015-04-21 6.623348 2015-04-22 6.468900 """ - ## price: This is the stitched price series - ## We can't use the price of the contract we're trading, or the volatility will be jumpy - ## And we'll miss out on the rolldown. See http://qoppac.blogspot.co.uk/2015/05/systems-building-futures-rolling.html - - ## We don't need to calculate the decay parameter, just use the span directly - - fast_ewma=pd.ewma(price, span=Lfast) - slow_ewma=pd.ewma(price, span=Lslow) - raw_ewmac=fast_ewma - slow_ewma - - return divide_df_single_column(raw_ewmac,vol) + # price: This is the stitched price series + # We can't use the price of the contract we're trading, or the volatility will be jumpy + # And we'll miss out on the rolldown. See + # http://qoppac.blogspot.co.uk/2015/05/systems-building-futures-rolling.html + + # We don't need to calculate the decay parameter, just use the span + # directly + + fast_ewma = pd.ewma(price, span=Lfast) + slow_ewma = pd.ewma(price, span=Lslow) + raw_ewmac = fast_ewma - slow_ewma + + return divide_df_single_column(raw_ewmac, vol) + def carry(daily_ann_roll, vol, smooth_days=90): """ Calculate raw carry forecast, given annualised roll and volatility series (which must match) - + Assumes that everything is daily data - :param daily_ann_roll: The annualised roll + :param daily_ann_roll: The annualised roll :type daily_ann_roll: pd.DataFrame (assumed Tx1) - :param vol: The daily price unit volatility (NOT % vol) + :param vol: The daily price unit volatility (NOT % vol) :type vol: pd.DataFrame (assumed Tx1) >>> from systems.provided.example.testdata import get_test_object_futures @@ -72,11 +76,11 @@ def carry(daily_ann_roll, vol, smooth_days=90): 2015-04-21 0.350892 2015-04-22 0.350892 """ - - ann_stdev=vol*ROOT_BDAYS_INYEAR - raw_carry=divide_df_single_column(daily_ann_roll,ann_stdev) - smooth_carry=pd.ewma(raw_carry, smooth_days) - + + ann_stdev = vol * ROOT_BDAYS_INYEAR + raw_carry = divide_df_single_column(daily_ann_roll, ann_stdev) + smooth_carry = pd.ewma(raw_carry, smooth_days) + return smooth_carry if __name__ == '__main__': diff --git a/systems/rawdata.py b/systems/rawdata.py index 97bc81b55c..11604c2f65 100644 --- a/systems/rawdata.py +++ b/systems/rawdata.py @@ -5,41 +5,41 @@ from syscore.objects import resolve_function from syscore.pdutils import divide_df_single_column + class RawData(SystemStage): """ A SystemStage that does some fairly common calculations before we do forecasting This is optional; forecasts can go straight to system.data - The advantages of using RawData are: + The advantages of using RawData are: - preliminary calculations that are reused can be cached, to save time (eg volatility) - preliminary calculations are available for inspection when diagnosing what is going on - KEY INPUT: system.data.get_instrument_price(instrument_code) + KEY INPUT: system.data.get_instrument_price(instrument_code) found in self.get_instrument_price - KEY OUTPUTS: system.rawdata.... several + KEY OUTPUTS: system.rawdata.... several Name: rawdata - """ - + """ + def __init__(self): """ Create a new stage: raw data object """ - setattr(self, "name", "rawdata") - + def get_instrument_price(self, instrument_code): """ Gets the instrument price from the parent system.data object - + KEY INPUT - - :param instrument_code: Instrument to get prices for + + :param instrument_code: Instrument to get prices for :type trading_rules: str - + :returns: Tx1 pd.DataFrame >>> from systems.tests.testdata import get_test_object @@ -53,21 +53,21 @@ def get_instrument_price(self, instrument_code): 2015-04-22 97.8325 """ def _get_instrument_price(system, instrument_code): - instrprice=system.data.get_instrument_price(instrument_code) + instrprice = system.data.get_instrument_price(instrument_code) return instrprice - - instrprice=self.parent.calc_or_cache( "instrument_price", instrument_code, _get_instrument_price) - + + instrprice = self.parent.calc_or_cache( + "instrument_price", instrument_code, _get_instrument_price) + return instrprice - - + def daily_prices(self, instrument_code): """ Gets daily prices - :param instrument_code: Instrument to get prices for + :param instrument_code: Instrument to get prices for :type trading_rules: str - + :returns: Tx1 pd.DataFrame KEY OUTPUT @@ -85,23 +85,23 @@ def daily_prices(self, instrument_code): """ def _daily_prices(system, instrument_code, this_stage): - instrprice=this_stage.get_instrument_price(instrument_code) - dailyprice=instrprice.resample("1B", how="last") + instrprice = this_stage.get_instrument_price(instrument_code) + dailyprice = instrprice.resample("1B", how="last") return dailyprice - - dailyprice=self.parent.calc_or_cache("daily_price", instrument_code, _daily_prices, self) - + + dailyprice = self.parent.calc_or_cache( + "daily_price", instrument_code, _daily_prices, self) + return dailyprice - - + def daily_denominator_price(self, instrument_code): """ Gets daily prices for use with % volatility This won't always be the same as the normal 'price' which is normally a cumulated total return series - :param instrument_code: Instrument to get prices for + :param instrument_code: Instrument to get prices for :type trading_rules: str - + :returns: Tx1 pd.DataFrame KEY OUTPUT @@ -118,24 +118,22 @@ def daily_denominator_price(self, instrument_code): 1983-09-27 71.306192 """ def _daily_denominator_returns(system, instrument_code, this_stage): - - dem_returns=this_stage.daily_prices(instrument_code) + + dem_returns = this_stage.daily_prices(instrument_code) return dem_returns - - dem_returns=self.parent.calc_or_cache( "daily_denominator_price", instrument_code, _daily_denominator_returns, self) - + + dem_returns = self.parent.calc_or_cache( + "daily_denominator_price", instrument_code, _daily_denominator_returns, self) + return dem_returns - - - - + def daily_returns(self, instrument_code): """ Gets daily returns (not % returns) - :param instrument_code: Instrument to get prices for + :param instrument_code: Instrument to get prices for :type trading_rules: str - + :returns: Tx1 pd.DataFrame KEY OUTPUT @@ -151,33 +149,32 @@ def daily_returns(self, instrument_code): 2015-04-22 -0.0725 """ def _daily_returns(system, instrument_code, this_stage): - instrdailyprice=this_stage.daily_prices(instrument_code) - dailyreturns=instrdailyprice.diff() + instrdailyprice = this_stage.daily_prices(instrument_code) + dailyreturns = instrdailyprice.diff() return dailyreturns - - dailyreturns=self.parent.calc_or_cache( "daily_returns", instrument_code, _daily_returns, self) - + + dailyreturns = self.parent.calc_or_cache( + "daily_returns", instrument_code, _daily_returns, self) + return dailyreturns - - - + def daily_returns_volatility(self, instrument_code): """ Gets volatility of daily returns (not % returns) - + This is done using a user defined function - + We get this from: the configuration object or if not found, system.defaults.py - + The dict must contain func key; anything else is optional KEY OUTPUT - :param instrument_code: Instrument to get prices for + :param instrument_code: Instrument to get prices for :type trading_rules: str - + :returns: Tx1 pd.DataFrame >>> from systems.tests.testdata import get_test_object @@ -205,46 +202,48 @@ def daily_returns_volatility(self, instrument_code): vol 2015-04-21 0.065903 2015-04-22 0.066014 - + """ - def _daily_returns_volatility(system, instrument_code, this_stage): - dailyreturns=this_stage.daily_returns(instrument_code) + def _daily_returns_volatility(system, instrument_code, this_stage): + dailyreturns = this_stage.daily_returns(instrument_code) try: - volconfig=copy(system.config.volatility_calculation) - identify_error="inherited from config object" + volconfig = copy(system.config.volatility_calculation) + identify_error = "inherited from config object" except: - volconfig=copy(system_defaults['volatility_calculation']) - identify_error="found in system.defaults.py" - + volconfig = copy(system_defaults['volatility_calculation']) + identify_error = "found in system.defaults.py" + if "func" not in volconfig: - - raise Exception("The volconfig dict (%s) needs to have a 'func' key" % identify_error) - - ## volconfig contains 'func' and some other arguments - ## we turn func which could be a string into a function, and then call it with the other ags - volfunction=resolve_function(volconfig.pop('func')) - vol=volfunction(dailyreturns, **volconfig) - + + raise Exception( + "The volconfig dict (%s) needs to have a 'func' key" % identify_error) + + # volconfig contains 'func' and some other arguments + # we turn func which could be a string into a function, and then + # call it with the other ags + volfunction = resolve_function(volconfig.pop('func')) + vol = volfunction(dailyreturns, **volconfig) + return vol - - vol=self.parent.calc_or_cache( "daily_returns_volatility", instrument_code, _daily_returns_volatility, self) - + + vol = self.parent.calc_or_cache( + "daily_returns_volatility", instrument_code, _daily_returns_volatility, self) + return vol - - + def get_daily_percentage_volatility(self, instrument_code): """ Get percentage returns normalised by recent vol - + Useful statistic, also used for some trading rules - + This is an optional subsystem; forecasts can go straight to system.data - :param instrument_code: Instrument to get prices for + :param instrument_code: Instrument to get prices for :type trading_rules: str - + :returns: Tx1 pd.DataFrame - + >>> from systems.tests.testdata import get_test_object >>> from systems.basesystem import System >>> @@ -255,27 +254,29 @@ def get_daily_percentage_volatility(self, instrument_code): 2015-04-21 0.058262 2015-04-22 0.059588 """ - def _get_daily_percentage_volatility(system, instrument_code, this_stage): - denom_price=this_stage.daily_denominator_price(instrument_code) - return_vol=this_stage.daily_returns_volatility(instrument_code) - perc_vol=100.0*divide_df_single_column(return_vol, denom_price.shift(1)) - + def _get_daily_percentage_volatility( + system, instrument_code, this_stage): + denom_price = this_stage.daily_denominator_price(instrument_code) + return_vol = this_stage.daily_returns_volatility(instrument_code) + perc_vol = 100.0 * \ + divide_df_single_column(return_vol, denom_price.shift(1)) + return perc_vol - - perc_vol=self.parent.calc_or_cache( "daily_percentage_volatility", instrument_code, _get_daily_percentage_volatility, self) - return perc_vol + perc_vol = self.parent.calc_or_cache( + "daily_percentage_volatility", instrument_code, _get_daily_percentage_volatility, self) + return perc_vol def norm_returns(self, instrument_code): """ Get returns normalised by recent vol - + Useful statistic, also used for some trading rules - + This is an optional subsystem; forecasts can go straight to system.data - :param instrument_code: Instrument to get prices for + :param instrument_code: Instrument to get prices for :type trading_rules: str - + :returns: Tx1 pd.DataFrame >>> from systems.tests.testdata import get_test_object @@ -289,17 +290,19 @@ def norm_returns(self, instrument_code): 2015-04-22 -1.270742 """ def _norm_returns(system, instrument_code, this_stage): - returnvol=this_stage.daily_returns_volatility(instrument_code).shift(1) - dailyreturns=this_stage.daily_returns(instrument_code) - norm_return=divide_df_single_column(dailyreturns,returnvol) - norm_return.columns=["norm_return"] + returnvol = this_stage.daily_returns_volatility( + instrument_code).shift(1) + dailyreturns = this_stage.daily_returns(instrument_code) + norm_return = divide_df_single_column(dailyreturns, returnvol) + norm_return.columns = ["norm_return"] return norm_return - - norm_returns=self.parent.calc_or_cache( "_norm_return_dict", instrument_code, _norm_returns, self) - + + norm_returns = self.parent.calc_or_cache( + "_norm_return_dict", instrument_code, _norm_returns, self) + return norm_returns - - + + if __name__ == '__main__': import doctest doctest.testmod() diff --git a/systems/stage.py b/systems/stage.py index ab190398c7..417807b2c6 100644 --- a/systems/stage.py +++ b/systems/stage.py @@ -1,34 +1,31 @@ - + class SystemStage(object): """ Default stage object: we inherit from this, rather than use 'in the raw' - + Here is the standard header to use for stages: - + Create a SystemStage for doing something - - + + KEY INPUT: system.....method(args) - found in self.method(args) - + found in self.method(args) + KEY OUTPUT: system.stage_name.method(args) Name: stage_name """ - + def __init__(self): ''' - + ''' - - ## We set these to empty lists but in the inherited object they'll be overriden + # We set these to empty lists but in the inherited object they'll be + # overriden setattr(self, "_protected", []) setattr(self, "name", "unnamed") - def __repr__(self): return "SystemStage '%s'" % self.name - - \ No newline at end of file diff --git a/systems/tests/test_base_systems.py b/systems/tests/test_base_systems.py index 47b47e6c62..40e45d3d95 100644 --- a/systems/tests/test_base_systems.py +++ b/systems/tests/test_base_systems.py @@ -7,76 +7,79 @@ from systems.stage import SystemStage from systems.basesystem import System, ALL_KEYNAME -class Test(unittest.TestCase): +class Test(unittest.TestCase): def testName(self): - stage=SystemStage() - stage.name="test" - stage._protected=["protected"] - - system=System([stage], None, None) + stage = SystemStage() + stage.name = "test" + stage._protected = ["protected"] + + system = System([stage], None, None) print(system._cache) - + system.set_item_in_cache(3, "a", "US10") print(system._cache) - + self.assertEqual(system.get_item_from_cache("a", "US10"), 3) - + def afunction(system, instrument_code, wibble, another_wibble=10): - return instrument_code+wibble+str(another_wibble) + return instrument_code + wibble + str(another_wibble) + + def anestedfunction(system, instrument_code, + keyname, wibble, another_wibble=10): + return instrument_code + wibble + keyname + str(another_wibble) - def anestedfunction(system, instrument_code, keyname, wibble, another_wibble=10): - return instrument_code+wibble+keyname+str(another_wibble) - - ans=system.calc_or_cache("b", "c", afunction, "d") + ans = system.calc_or_cache("b", "c", afunction, "d") print(system._cache) - ans=system.calc_or_cache("b", "c", afunction, "d", 20.0) + ans = system.calc_or_cache("b", "c", afunction, "d", 20.0) print(system._cache) - ans=system.calc_or_cache_nested("c", "SP500", "thing", anestedfunction, "k") + ans = system.calc_or_cache_nested( + "c", "SP500", "thing", anestedfunction, "k") print(system._cache) - ans=system.calc_or_cache("b", ALL_KEYNAME, afunction, "e", 120.0) + ans = system.calc_or_cache("b", ALL_KEYNAME, afunction, "e", 120.0) print(system._cache) - - ans=system.calc_or_cache("protected", ALL_KEYNAME, afunction, "e", 120.0) - - ans=system.get_item_from_cache("c", "SP500", "thing") + + ans = system.calc_or_cache( + "protected", ALL_KEYNAME, afunction, "e", 120.0) + + ans = system.get_item_from_cache("c", "SP500", "thing") print(system._cache) system._delete_item_from_cache("b", "c") print(system._cache) - ans=system.set_item_in_cache(10.0, "protected", "SP500") + ans = system.set_item_in_cache(10.0, "protected", "SP500") print(system._cache) print(system.get_item_from_cache("b", ALL_KEYNAME)) - + print(system.get_items_with_all()) print(system.get_items_with_data()) print(system.get_protected_items()) print(system.get_items_for_instrument("SP500")) - + system.delete_items_with_all() print(system._cache) - + system.delete_items_with_all(True) - + print(system._cache) - + system.delete_items_for_instrument("SP500") print(system._cache) - + system.delete_items_for_instrument("SP500", True) print(system._cache) - + system.delete_item("a") - + print(system._cache) - + if __name__ == "__main__": #import sys;sys.argv = ['', 'Test.testName'] - unittest.main() \ No newline at end of file + unittest.main() diff --git a/systems/tests/test_forecasts.py b/systems/tests/test_forecasts.py index 58467de28b..9ae364396a 100644 --- a/systems/tests/test_forecasts.py +++ b/systems/tests/test_forecasts.py @@ -5,152 +5,164 @@ ''' import unittest from systems.provided.example.rules import ewmac_forecast_with_defaults -from systems.forecasting import TradingRule, Rules, process_trading_rules, create_variations, create_variations_oneparameter +from systems.forecasting import TradingRule, Rules, process_trading_rules, create_variations, create_variations_oneparameter from systems.basesystem import System from systems.rawdata import RawData from sysdata.configdata import Config from sysdata.csvdata import csvFuturesData - class Test(unittest.TestCase): def testRules(self): - - #config=Config(dict(trading_rules=dict(ewmac=dict(function="systems.provided.example.rules.ewmac_forecast_with_defaults")))) - data=csvFuturesData("sysdata.tests") - - rules=Rules(dict(function="systems.provided.example.rules.ewmac_forecast_with_defaults")) - system=System([rules], data) - - ans=system.rules.get_raw_forecast("EDOLLAR", "rule0") + + # config=Config(dict(trading_rules=dict(ewmac=dict(function="systems.provided.example.rules.ewmac_forecast_with_defaults")))) + data = csvFuturesData("sysdata.tests") + + rules = Rules( + dict(function="systems.provided.example.rules.ewmac_forecast_with_defaults")) + system = System([rules], data) + + ans = system.rules.get_raw_forecast("EDOLLAR", "rule0") self.assertAlmostEqual(ans.iloc[-1][0], 4.0410786602, 5) - - config=Config(dict(trading_rules=dict(ewmac=dict(function="systems.provided.example.rules.ewmac_forecast_with_defaults")))) - rules=Rules() - system=System([rules], data, config) - ans=system.rules.get_raw_forecast("EDOLLAR", "ewmac") + + config = Config(dict(trading_rules=dict(ewmac=dict( + function="systems.provided.example.rules.ewmac_forecast_with_defaults")))) + rules = Rules() + system = System([rules], data, config) + ans = system.rules.get_raw_forecast("EDOLLAR", "ewmac") self.assertAlmostEqual(ans.iloc[-1][0], 4.0410786602, 5) - - config=Config("systems.provided.example.exampleconfig.yaml") - rawdata=RawData() - - rules=Rules() - system=System([rules, rawdata], data, config) - ans=system.rules.get_raw_forecast("EDOLLAR", "ewmac8") + + config = Config("systems.provided.example.exampleconfig.yaml") + rawdata = RawData() + + rules = Rules() + system = System([rules, rawdata], data, config) + ans = system.rules.get_raw_forecast("EDOLLAR", "ewmac8") self.assertAlmostEqual(ans.iloc[-1][0], 0.954941033, 5) - + def testinitTradingRules(self): - rule=TradingRule((ewmac_forecast_with_defaults, [], {})) - assert rule.function==ewmac_forecast_with_defaults - rule2=TradingRule(rule) - - assert (rule.function, rule.data, rule.other_args)==(rule2.function, rule2.data, rule2.other_args) - - rule3=TradingRule(ewmac_forecast_with_defaults, ["data.get_instrument_price"], dict(Lfast=50, Lslow=200)) - - assert rule3.data==["data.get_instrument_price"] - - rule4=TradingRule(ewmac_forecast_with_defaults, "data.get_instrument_price", dict(Lfast=50, Lslow=200)) - - assert rule4.data==["data.get_instrument_price"] - + rule = TradingRule((ewmac_forecast_with_defaults, [], {})) + assert rule.function == ewmac_forecast_with_defaults + rule2 = TradingRule(rule) + + assert (rule.function, rule.data, rule.other_args) == ( + rule2.function, rule2.data, rule2.other_args) + + rule3 = TradingRule(ewmac_forecast_with_defaults, [ + "data.get_instrument_price"], dict(Lfast=50, Lslow=200)) + + assert rule3.data == ["data.get_instrument_price"] + + rule4 = TradingRule(ewmac_forecast_with_defaults, + "data.get_instrument_price", dict(Lfast=50, Lslow=200)) + + assert rule4.data == ["data.get_instrument_price"] + try: - rule4=TradingRule(ewmac_forecast_with_defaults, "data.get_instrument_price") - rule5=TradingRule((ewmac_forecast_with_defaults, )) + rule4 = TradingRule(ewmac_forecast_with_defaults, + "data.get_instrument_price") + rule5 = TradingRule((ewmac_forecast_with_defaults, )) raise Exception("Should have failed with 2 tuple") except: pass - - rule7=TradingRule("systems.provided.example.rules.ewmac_forecast_with_defaults", [], dict(Lfast=50, Lslow=200)) - assert rule7.function==rule.function - + + rule7 = TradingRule("systems.provided.example.rules.ewmac_forecast_with_defaults", [ + ], dict(Lfast=50, Lslow=200)) + assert rule7.function == rule.function + try: - rule8=TradingRule("not.a.real.rule.just.a.string.of.arbitrary.text", [], dict(Lfast=50, Lslow=200)) + rule8 = TradingRule("not.a.real.rule.just.a.string.of.arbitrary.text", [ + ], dict(Lfast=50, Lslow=200)) raise Exception("Should have failed with non existent rule") except: pass - - rule9=TradingRule(dict(function=ewmac_forecast_with_defaults)) - + + rule9 = TradingRule(dict(function=ewmac_forecast_with_defaults)) + try: - rule10=TradingRule(dict(functionette=ewmac_forecast_with_defaults)) + rule10 = TradingRule( + dict(functionette=ewmac_forecast_with_defaults)) raise Exception("Should have failed with no function keyword") except: pass - rule11=TradingRule(dict(function="systems.provided.example.rules.ewmac_forecast_with_defaults", - other_args=dict(Lfast=50), data=[])) - - rule12=TradingRule(ewmac_forecast_with_defaults, other_args=dict(Lfast=30)) - rule13=TradingRule("systems.provided.example.rules.ewmac_forecast_with_defaults", data="data.get_pricedata") - assert rule13.data==["data.get_pricedata"] - - rule14=TradingRule(ewmac_forecast_with_defaults) - + rule11 = TradingRule(dict(function="systems.provided.example.rules.ewmac_forecast_with_defaults", + other_args=dict(Lfast=50), data=[])) + + rule12 = TradingRule(ewmac_forecast_with_defaults, + other_args=dict(Lfast=30)) + rule13 = TradingRule( + "systems.provided.example.rules.ewmac_forecast_with_defaults", data="data.get_pricedata") + assert rule13.data == ["data.get_pricedata"] + + rule14 = TradingRule(ewmac_forecast_with_defaults) + try: - rule15=TradingRule(set()) + rule15 = TradingRule(set()) raise Exception("Should have failed with other data type") except: pass def testCallingTradingRule(self): - - #config=Config(dict(trading_rules=dict(ewmac=dict(function="systems.provided.example.rules.ewmac_forecast_with_defaults")))) - data=csvFuturesData("sysdata.tests") - - rawdata=RawData() - rules=Rules() - system=System([rawdata, rules], data) - - ## Call with default data and config - rule=TradingRule(ewmac_forecast_with_defaults) - ans=rule.call(system, "EDOLLAR") + + # config=Config(dict(trading_rules=dict(ewmac=dict(function="systems.provided.example.rules.ewmac_forecast_with_defaults")))) + data = csvFuturesData("sysdata.tests") + + rawdata = RawData() + rules = Rules() + system = System([rawdata, rules], data) + + # Call with default data and config + rule = TradingRule(ewmac_forecast_with_defaults) + ans = rule.call(system, "EDOLLAR") self.assertAlmostEqual(ans.iloc[-1][0], 4.0410786602, 5) - - ## Change the data source - rule=TradingRule(("systems.provided.example.rules.ewmac_forecast_with_defaults_no_vol", - ["rawdata.daily_prices", "rawdata.daily_returns_volatility"], dict())) - ans=rule.call(system, "EDOLLAR") + # Change the data source + rule = TradingRule(("systems.provided.example.rules.ewmac_forecast_with_defaults_no_vol", + ["rawdata.daily_prices", "rawdata.daily_returns_volatility"], dict())) + + ans = rule.call(system, "EDOLLAR") self.assertAlmostEqual(ans.iloc[-1][0], 0.954941033, 5) - - rule=TradingRule(dict(function="systems.provided.example.rules.ewmac_forecast_with_defaults_no_vol", - data=["rawdata.daily_prices", "rawdata.daily_returns_volatility"], - other_args=dict(Lfast=50, Lslow=200))) - ans=rule.call(system, "EDOLLAR") + + rule = TradingRule(dict(function="systems.provided.example.rules.ewmac_forecast_with_defaults_no_vol", + data=["rawdata.daily_prices", + "rawdata.daily_returns_volatility"], + other_args=dict(Lfast=50, Lslow=200))) + ans = rule.call(system, "EDOLLAR") self.assertAlmostEqual(ans.iloc[-1][0], 5.603808, 5) - + def testProcessTradingRuleSpec(self): - - ruleA=TradingRule(ewmac_forecast_with_defaults) - ruleB=TradingRule(dict(function="systems.provided.example.rules.ewmac_forecast_with_defaults_no_vol", - data=["rawdata.daily_prices", "rawdata.daily_returns_volatility"], - other_args=dict(Lfast=50, Lslow=200))) - - - trading_rules=dict(ruleA=ruleA, ruleB=ruleB) - ans=process_trading_rules(trading_rules) + + ruleA = TradingRule(ewmac_forecast_with_defaults) + ruleB = TradingRule(dict(function="systems.provided.example.rules.ewmac_forecast_with_defaults_no_vol", + data=["rawdata.daily_prices", + "rawdata.daily_returns_volatility"], + other_args=dict(Lfast=50, Lslow=200))) + + trading_rules = dict(ruleA=ruleA, ruleB=ruleB) + ans = process_trading_rules(trading_rules) assert "ruleA" in ans.keys() assert "ruleB" in ans.keys() - trading_rules=[ruleA, ruleB] - ans=process_trading_rules(trading_rules) + trading_rules = [ruleA, ruleB] + ans = process_trading_rules(trading_rules) assert "rule1" in ans.keys() - - ans=process_trading_rules(ruleA) - assert ans['rule0'].function==ewmac_forecast_with_defaults - - ans=process_trading_rules(ewmac_forecast_with_defaults) - assert ans['rule0'].function==ewmac_forecast_with_defaults - - ans=process_trading_rules([dict(function="systems.provided.example.rules.ewmac_forecast_with_defaults_no_vol", - data=["rawdata.daily_prices", "rawdata.daily_returns_volatility"], - other_args=dict(Lfast=50, Lslow=200))]) - assert ans['rule0'].other_args['Lfast']==50 - - + + ans = process_trading_rules(ruleA) + assert ans['rule0'].function == ewmac_forecast_with_defaults + + ans = process_trading_rules(ewmac_forecast_with_defaults) + assert ans['rule0'].function == ewmac_forecast_with_defaults + + ans = process_trading_rules([dict(function="systems.provided.example.rules.ewmac_forecast_with_defaults_no_vol", + data=["rawdata.daily_prices", + "rawdata.daily_returns_volatility"], + other_args=dict(Lfast=50, Lslow=200))]) + assert ans['rule0'].other_args['Lfast'] == 50 + + if __name__ == "__main__": #import sys;sys.argv = ['', 'Test.testTradingRules'] - unittest.main() \ No newline at end of file + unittest.main() diff --git a/systems/tests/testdata.py b/systems/tests/testdata.py index be41f770d4..8796a51c2c 100644 --- a/systems/tests/testdata.py +++ b/systems/tests/testdata.py @@ -1,5 +1,5 @@ -from systems.futures.rawdata import FuturesRawData +from systems.futures.rawdata import FuturesRawData from systems.rawdata import RawData from sysdata.csvdata import csvFuturesData from sysdata.configdata import Config @@ -8,69 +8,72 @@ from systems.forecast_combine import ForecastCombineFixed from systems.positionsizing import PositionSizing + def get_test_object(): """ Returns some standard test data """ - data=csvFuturesData("sysdata.tests") - rawdata=RawData() - config=Config("systems.provided.example.exampleconfig.yaml") - return (rawdata, data, config) + data = csvFuturesData("sysdata.tests") + rawdata = RawData() + config = Config("systems.provided.example.exampleconfig.yaml") + return (rawdata, data, config) def get_test_object_futures(): """ Returns some standard test data """ - data=csvFuturesData("sysdata.tests") - rawdata=FuturesRawData() - config=Config("systems.provided.example.exampleconfig.yaml") - return (rawdata, data, config) + data = csvFuturesData("sysdata.tests") + rawdata = FuturesRawData() + config = Config("systems.provided.example.exampleconfig.yaml") + return (rawdata, data, config) def get_test_object_futures_with_rules(): """ Returns some standard test data """ - data=csvFuturesData("sysdata.tests") - rawdata=FuturesRawData() - rules=Rules() - config=Config("systems.provided.example.exampleconfig.yaml") - return (rules, rawdata, data, config) + data = csvFuturesData("sysdata.tests") + rawdata = FuturesRawData() + rules = Rules() + config = Config("systems.provided.example.exampleconfig.yaml") + return (rules, rawdata, data, config) + def get_test_object_futures_with_rules_and_capping(): """ Returns some standard test data """ - data=csvFuturesData("sysdata.tests") - rawdata=FuturesRawData() - rules=Rules() - config=Config("systems.provided.example.exampleconfig.yaml") - capobject=ForecastScaleCapFixed() - return (capobject, rules, rawdata, data, config) + data = csvFuturesData("sysdata.tests") + rawdata = FuturesRawData() + rules = Rules() + config = Config("systems.provided.example.exampleconfig.yaml") + capobject = ForecastScaleCapFixed() + return (capobject, rules, rawdata, data, config) + def get_test_object_futures_with_comb_forecasts(): """ Returns some standard test data """ - data=csvFuturesData("sysdata.tests") - rawdata=FuturesRawData() - rules=Rules() - config=Config("systems.provided.example.exampleconfig.yaml") - capobject=ForecastScaleCapFixed() - combobject=ForecastCombineFixed() - return (combobject, capobject, rules, rawdata, data, config) + data = csvFuturesData("sysdata.tests") + rawdata = FuturesRawData() + rules = Rules() + config = Config("systems.provided.example.exampleconfig.yaml") + capobject = ForecastScaleCapFixed() + combobject = ForecastCombineFixed() + return (combobject, capobject, rules, rawdata, data, config) + def get_test_object_futures_with_pos_sizing(): """ Returns some standard test data """ - data=csvFuturesData("sysdata.tests") - rawdata=FuturesRawData() - rules=Rules() - config=Config("systems.provided.example.exampleconfig.yaml") - capobject=ForecastScaleCapFixed() - combobject=ForecastCombineFixed() - posobject=PositionSizing() - return (posobject, combobject, capobject, rules, rawdata, data, config) - + data = csvFuturesData("sysdata.tests") + rawdata = FuturesRawData() + rules = Rules() + config = Config("systems.provided.example.exampleconfig.yaml") + capobject = ForecastScaleCapFixed() + combobject = ForecastCombineFixed() + posobject = PositionSizing() + return (posobject, combobject, capobject, rules, rawdata, data, config)