Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

update .ipynb format check #2

Merged
merged 7 commits into from
Feb 16, 2023
Merged

update .ipynb format check #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
3e45075
modify nbqa check path
Cadenza-Li Feb 15, 2023
d8e1f67
add pylint flake8 mypy check to ipynb
Cadenza-Li Feb 15, 2023
b33b0c6
check ipynb with black and pylint
Cadenza-Li Feb 15, 2023
bd0c6a8
reformat .ipynb files
Cadenza-Li Feb 15, 2023
37e5605
format line length
Cadenza-Li Feb 15, 2023
90df07e
update nbqa .ipynb format CI
Cadenza-Li Feb 15, 2023
e79e3c9
format old ipynb files
Cadenza-Li Feb 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/workflows/test_qlib_from_source.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,8 @@ jobs:

- name: Check Qlib ipynb with nbqa
run: |
nbqa black qlib
nbqa black . -l 120 --check --diff
nbqa pylint . --disable=C0104,C0114,C0115,C0116,C0301,C0302,C0411,C0413,C1802,R0401,R0801,R0902,R0903,R0911,R0912,R0913,R0914,R0915,R1720,W0105,W0123,W0201,W0511,W0613,W1113,W1514,E0401,E1121,C0103,C0209,R0402,R1705,R1710,R1725,R1735,W0102,W0212,W0221,W0223,W0231,W0237,W0612,W0621,W0622,W0703,W1309,E1102,E1136,W0719,W0104,W0404,C0412,W0611,C0410 --const-rgx='[a-z_][a-z0-9_]{2,30}$'

- name: Test data downloads
run: |
Expand Down
232 changes: 131 additions & 101 deletions examples/benchmarks/TRA/Reports.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,59 +25,65 @@
"import seaborn as sns\n",
"import matplotlib.pyplot as plt\n",
"import matplotlib\n",
"sns.set(style='white')\n",
"matplotlib.rcParams['pdf.fonttype'] = 42\n",
"matplotlib.rcParams['ps.fonttype'] = 42\n",
"\n",
"sns.set(style=\"white\")\n",
"matplotlib.rcParams[\"pdf.fonttype\"] = 42\n",
"matplotlib.rcParams[\"ps.fonttype\"] = 42\n",
"\n",
"from tqdm.auto import tqdm\n",
"from joblib import Parallel, delayed\n",
"\n",
"\n",
"def func(x, N=80):\n",
" ret = x.ret.copy()\n",
" x = x.rank(pct=True)\n",
" x['ret'] = ret\n",
" x[\"ret\"] = ret\n",
" diff = x.score.sub(x.label)\n",
" r = x.nlargest(N, columns='score').ret.mean()\n",
" r -= x.nsmallest(N, columns='score').ret.mean()\n",
" return pd.Series({\n",
" 'MSE': diff.pow(2).mean(), \n",
" 'MAE': diff.abs().mean(), \n",
" 'IC': x.score.corr(x.label),\n",
" 'R': r\n",
" })\n",
" \n",
" r = x.nlargest(N, columns=\"score\").ret.mean()\n",
" r -= x.nsmallest(N, columns=\"score\").ret.mean()\n",
" return pd.Series(\n",
" {\n",
" \"MSE\": diff.pow(2).mean(),\n",
" \"MAE\": diff.abs().mean(),\n",
" \"IC\": x.score.corr(x.label),\n",
" \"R\": r,\n",
" }\n",
" )\n",
"\n",
"\n",
"ret = pd.read_pickle(\"data/ret.pkl\").clip(-0.1, 0.1)\n",
"\n",
"\n",
"def backtest(fname, **kwargs):\n",
" pred = pd.read_pickle(fname).loc['2018-09-21':'2020-06-30'] # test period\n",
" pred['ret'] = ret\n",
" pred = pd.read_pickle(fname).loc[\"2018-09-21\":\"2020-06-30\"] # test period\n",
" pred[\"ret\"] = ret\n",
" dates = pred.index.unique(level=0)\n",
" res = Parallel(n_jobs=-1)(delayed(func)(pred.loc[d], **kwargs) for d in dates)\n",
" res = {\n",
" dates[i]: res[i]\n",
" for i in range(len(dates))\n",
" }\n",
" res = {dates[i]: res[i] for i in range(len(dates))}\n",
" res = pd.DataFrame(res).T\n",
" r = res['R'].copy()\n",
" r = res[\"R\"].copy()\n",
" r.index = pd.to_datetime(r.index)\n",
" r = r.reindex(pd.date_range(r.index[0], r.index[-1])).fillna(0) # paper use 365 days\n",
" return {\n",
" 'MSE': res['MSE'].mean(),\n",
" 'MAE': res['MAE'].mean(),\n",
" 'IC': res['IC'].mean(),\n",
" 'ICIR': res['IC'].mean()/res['IC'].std(),\n",
" 'AR': r.mean()*365,\n",
" 'AV': r.std()*365**0.5,\n",
" 'SR': r.mean()/r.std()*365**0.5,\n",
" 'MDD': (r.cumsum().cummax() - r.cumsum()).max()\n",
" \"MSE\": res[\"MSE\"].mean(),\n",
" \"MAE\": res[\"MAE\"].mean(),\n",
" \"IC\": res[\"IC\"].mean(),\n",
" \"ICIR\": res[\"IC\"].mean() / res[\"IC\"].std(),\n",
" \"AR\": r.mean() * 365,\n",
" \"AV\": r.std() * 365**0.5,\n",
" \"SR\": r.mean() / r.std() * 365**0.5,\n",
" \"MDD\": (r.cumsum().cummax() - r.cumsum()).max(),\n",
" }, r\n",
"\n",
"\n",
"def fmt(x, p=3, scale=1, std=False):\n",
" _fmt = '{:.%df}'%p\n",
" _fmt = \"{:.%df}\" % p\n",
" string = _fmt.format((x.mean() if not isinstance(x, (float, np.floating)) else x) * scale)\n",
" if std and len(x) > 1:\n",
" string += ' ('+_fmt.format(x.std()*scale)+')'\n",
" string += \" (\" + _fmt.format(x.std() * scale) + \")\"\n",
" return string\n",
"\n",
"\n",
"def backtest_multi(files, **kwargs):\n",
" res = []\n",
" pnl = []\n",
Expand All @@ -88,14 +94,14 @@
" res = pd.DataFrame(res)\n",
" pnl = pd.concat(pnl, axis=1)\n",
" return {\n",
" 'MSE': fmt(res['MSE'], std=True),\n",
" 'MAE': fmt(res['MAE'], std=True),\n",
" 'IC': fmt(res['IC']),\n",
" 'ICIR': fmt(res['ICIR']),\n",
" 'AR': fmt(res['AR'], scale=100, p=1)+'%',\n",
" 'VR': fmt(res['AV'], scale=100, p=1)+'%',\n",
" 'SR': fmt(res['SR']),\n",
" 'MDD': fmt(res['MDD'], scale=100, p=1)+'%'\n",
" \"MSE\": fmt(res[\"MSE\"], std=True),\n",
" \"MAE\": fmt(res[\"MAE\"], std=True),\n",
" \"IC\": fmt(res[\"IC\"]),\n",
" \"ICIR\": fmt(res[\"ICIR\"]),\n",
" \"AR\": fmt(res[\"AR\"], scale=100, p=1) + \"%\",\n",
" \"VR\": fmt(res[\"AV\"], scale=100, p=1) + \"%\",\n",
" \"SR\": fmt(res[\"SR\"]),\n",
" \"MDD\": fmt(res[\"MDD\"], scale=100, p=1) + \"%\",\n",
" }, pnl"
]
},
Expand Down Expand Up @@ -124,16 +130,20 @@
"outputs": [],
"source": [
"exps = {\n",
" 'Linear': ['output/Linear/pred.pkl'],\n",
" 'LightGBM': ['output/GBDT/lr0.05_leaves128/pred.pkl'],\n",
" 'MLP': glob.glob('output/search/MLP/hs128_bs512_do0.3_lr0.001_seed*/pred.pkl'),\n",
" 'SFM': glob.glob('output/search/SFM/hs32_bs512_do0.5_lr0.001_seed*/pred.pkl'),\n",
" 'ALSTM': glob.glob('output/search/LSTM_Attn/hs256_bs1024_do0.1_lr0.0002_seed*/pred.pkl'),\n",
" 'Trans.': glob.glob('output/search/Transformer/head4_hs64_bs1024_do0.1_lr0.0002_seed*/pred.pkl'),\n",
" 'ALSTM+TS':glob.glob('output/LSTM_Attn_TS/hs256_bs1024_do0.1_lr0.0002_seed*/pred.pkl'),\n",
" 'Trans.+TS':glob.glob('output/Transformer_TS/head4_hs64_bs1024_do0.1_lr0.0002_seed*/pred.pkl'),\n",
" 'ALSTM+TRA(Ours)': glob.glob('output/search/finetune/LSTM_Attn_tra/K10_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/pred.pkl'),\n",
" 'Trans.+TRA(Ours)': glob.glob('output/search/finetune/Transformer_tra/K3_traHs16_traSrcLR_TPE_traLamb1.0_head4_hs64_bs512_do0.1_lr0.0005_seed*/pred.pkl')\n",
" \"Linear\": [\"output/Linear/pred.pkl\"],\n",
" \"LightGBM\": [\"output/GBDT/lr0.05_leaves128/pred.pkl\"],\n",
" \"MLP\": glob.glob(\"output/search/MLP/hs128_bs512_do0.3_lr0.001_seed*/pred.pkl\"),\n",
" \"SFM\": glob.glob(\"output/search/SFM/hs32_bs512_do0.5_lr0.001_seed*/pred.pkl\"),\n",
" \"ALSTM\": glob.glob(\"output/search/LSTM_Attn/hs256_bs1024_do0.1_lr0.0002_seed*/pred.pkl\"),\n",
" \"Trans.\": glob.glob(\"output/search/Transformer/head4_hs64_bs1024_do0.1_lr0.0002_seed*/pred.pkl\"),\n",
" \"ALSTM+TS\": glob.glob(\"output/LSTM_Attn_TS/hs256_bs1024_do0.1_lr0.0002_seed*/pred.pkl\"),\n",
" \"Trans.+TS\": glob.glob(\"output/Transformer_TS/head4_hs64_bs1024_do0.1_lr0.0002_seed*/pred.pkl\"),\n",
" \"ALSTM+TRA(Ours)\": glob.glob(\n",
" \"output/search/finetune/LSTM_Attn_tra/K10_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/pred.pkl\"\n",
" ),\n",
" \"Trans.+TRA(Ours)\": glob.glob(\n",
" \"output/search/finetune/Transformer_tra/K3_traHs16_traSrcLR_TPE_traLamb1.0_head4_hs64_bs512_do0.1_lr0.0005_seed*/pred.pkl\"\n",
" ),\n",
"}"
]
},
Expand All @@ -160,14 +170,8 @@
}
],
"source": [
"res = {\n",
" name: backtest_multi(exps[name])\n",
" for name in tqdm(exps)\n",
"}\n",
"report = pd.DataFrame({\n",
" k: v[0]\n",
" for k, v in res.items()\n",
"}).T"
"res = {name: backtest_multi(exps[name]) for name in tqdm(exps)}\n",
"report = pd.DataFrame({k: v[0] for k, v in res.items()}).T"
]
},
{
Expand Down Expand Up @@ -385,24 +389,40 @@
}
],
"source": [
"df = pd.read_pickle('output/search/finetune/Transformer_tra/K3_traHs16_traSrcLR_TPE_traLamb0.0_head4_hs64_bs512_do0.1_lr0.0005_seed1000/pred.pkl')\n",
"code = 'SH600157'\n",
"date = '2018-09-28'\n",
"df = pd.read_pickle(\n",
" \"output/search/finetune/Transformer_tra/K3_traHs16_traSrcLR_TPE_traLamb0.0_head4_hs64_bs512_do0.1_lr0.0005_seed1000/pred.pkl\"\n",
")\n",
"code = \"SH600157\"\n",
"date = \"2018-09-28\"\n",
"lookbackperiod = 50\n",
"\n",
"prob = df.iloc[:, -3:].loc(axis=0)[:, code].reset_index(level=1, drop=True).loc[date:].iloc[:lookbackperiod]\n",
"pred = df.loc[:,[\"score_0\",\"score_1\",\"score_2\",\"label\"]].loc(axis=0)[:, code].reset_index(level=1, drop=True).loc[date:].iloc[:lookbackperiod]\n",
"e_all = pred.iloc[:,:-1].sub(pred.iloc[:,-1], axis=0).pow(2)\n",
"pred = (\n",
" df.loc[:, [\"score_0\", \"score_1\", \"score_2\", \"label\"]]\n",
" .loc(axis=0)[:, code]\n",
" .reset_index(level=1, drop=True)\n",
" .loc[date:]\n",
" .iloc[:lookbackperiod]\n",
")\n",
"e_all = pred.iloc[:, :-1].sub(pred.iloc[:, -1], axis=0).pow(2)\n",
"e_all = e_all.sub(e_all.min(axis=1), axis=0)\n",
"e_all.columns = [r'$\\theta_%d$'%d for d in range(1, 4)]\n",
"e_all.columns = [r\"$\\theta_%d$\" % d for d in range(1, 4)]\n",
"prob = pd.Series(np.argmax(prob.values, axis=1), index=prob.index).rolling(7).mean().round()\n",
"\n",
"fig, axes = plt.subplots(1, 2, figsize=(7, 3))\n",
"e_all.plot(ax=axes[0], xlabel='', rot=30)\n",
"prob.plot(ax=axes[1], xlabel='', rot=30, color='red', linestyle='None', marker='^', markersize=5)\n",
"e_all.plot(ax=axes[0], xlabel=\"\", rot=30)\n",
"prob.plot(\n",
" ax=axes[1],\n",
" xlabel=\"\",\n",
" rot=30,\n",
" color=\"red\",\n",
" linestyle=\"None\",\n",
" marker=\"^\",\n",
" markersize=5,\n",
")\n",
"plt.yticks(np.array([0, 1, 2]), e_all.columns.values)\n",
"axes[0].set_ylabel('Predictor Loss')\n",
"axes[1].set_ylabel('Router Selection')\n",
"axes[0].set_ylabel(\"Predictor Loss\")\n",
"axes[1].set_ylabel(\"Router Selection\")\n",
"plt.tight_layout()\n",
"# plt.savefig('select.pdf', bbox_inches='tight')\n",
"plt.show()"
Expand All @@ -428,10 +448,18 @@
"outputs": [],
"source": [
"exps = {\n",
" 'Random': glob.glob('output/search/LSTM_Attn_tra/K10_traHs16_traSrcNONE_traLamb1.0_hs256_bs1024_do0.1_lr0.0001_seed*/pred.pkl'),\n",
" 'LR': glob.glob('output/search/LSTM_Attn_tra/K10_traHs16_traSrcLR_traLamb1.0_hs256_bs1024_do0.1_lr0.0001_seed*/pred.pkl'),\n",
" 'TPE': glob.glob('output/search/LSTM_Attn_tra/K10_traHs16_traSrcTPE_traLamb1.0_hs256_bs1024_do0.1_lr0.0001_seed*/pred.pkl'),\n",
" 'LR+TPE': glob.glob('output/search/finetune/LSTM_Attn_tra/K10_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/pred.pkl')\n",
" \"Random\": glob.glob(\n",
" \"output/search/LSTM_Attn_tra/K10_traHs16_traSrcNONE_traLamb1.0_hs256_bs1024_do0.1_lr0.0001_seed*/pred.pkl\"\n",
" ),\n",
" \"LR\": glob.glob(\n",
" \"output/search/LSTM_Attn_tra/K10_traHs16_traSrcLR_traLamb1.0_hs256_bs1024_do0.1_lr0.0001_seed*/pred.pkl\"\n",
" ),\n",
" \"TPE\": glob.glob(\n",
" \"output/search/LSTM_Attn_tra/K10_traHs16_traSrcTPE_traLamb1.0_hs256_bs1024_do0.1_lr0.0001_seed*/pred.pkl\"\n",
" ),\n",
" \"LR+TPE\": glob.glob(\n",
" \"output/search/finetune/LSTM_Attn_tra/K10_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/pred.pkl\"\n",
" ),\n",
"}"
]
},
Expand All @@ -456,14 +484,8 @@
}
],
"source": [
"res = {\n",
" name: backtest_multi(exps[name])\n",
" for name in tqdm(exps)\n",
"}\n",
"report = pd.DataFrame({\n",
" k: v[0]\n",
" for k, v in res.items()\n",
"}).T"
"res = {name: backtest_multi(exps[name]) for name in tqdm(exps)}\n",
"report = pd.DataFrame({k: v[0] for k, v in res.items()}).T"
]
},
{
Expand Down Expand Up @@ -597,18 +619,22 @@
}
],
"source": [
"a = pd.read_pickle('output/search/finetune/Transformer_tra/K3_traHs16_traSrcLR_TPE_traLamb0.0_head4_hs64_bs512_do0.1_lr0.0005_seed3000/pred.pkl')\n",
"b = pd.read_pickle('output/search/finetune/Transformer_tra/K3_traHs16_traSrcLR_TPE_traLamb2.0_head4_hs64_bs512_do0.1_lr0.0005_seed3000/pred.pkl')\n",
"a = pd.read_pickle(\n",
" \"output/search/finetune/Transformer_tra/K3_traHs16_traSrcLR_TPE_traLamb0.0_head4_hs64_bs512_do0.1_lr0.0005_seed3000/pred.pkl\"\n",
")\n",
"b = pd.read_pickle(\n",
" \"output/search/finetune/Transformer_tra/K3_traHs16_traSrcLR_TPE_traLamb2.0_head4_hs64_bs512_do0.1_lr0.0005_seed3000/pred.pkl\"\n",
")\n",
"a = a.iloc[:, -3:]\n",
"b = b.iloc[:, -3:]\n",
"b = np.eye(3)[b.values.argmax(axis=1)]\n",
"a = np.eye(3)[a.values.argmax(axis=1)]\n",
"\n",
"res = pd.DataFrame({\n",
" 'with OT': b.sum(axis=0) / b.sum(),\n",
" 'without OT': a.sum(axis=0)/ a.sum() \n",
"},index=[r'$\\theta_1$',r'$\\theta_2$',r'$\\theta_3$'])\n",
"res.plot.bar(rot=30, figsize=(5, 4), color=['b', 'g'])\n",
"res = pd.DataFrame(\n",
" {\"with OT\": b.sum(axis=0) / b.sum(), \"without OT\": a.sum(axis=0) / a.sum()},\n",
" index=[r\"$\\theta_1$\", r\"$\\theta_2$\", r\"$\\theta_3$\"],\n",
")\n",
"res.plot.bar(rot=30, figsize=(5, 4), color=[\"b\", \"g\"])\n",
"del a, b"
]
},
Expand All @@ -633,11 +659,19 @@
"outputs": [],
"source": [
"exps = {\n",
" 'K=1': glob.glob('output/search/LSTM_Attn/hs256_bs1024_do0.1_lr0.0002_seed*/info.json'),\n",
" 'K=3': glob.glob('output/search/finetune/LSTM_Attn_tra/K3_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/info.json'),\n",
" 'K=5': glob.glob('output/search/finetune/LSTM_Attn_tra/K5_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/info.json'),\n",
" 'K=10': glob.glob('output/search/finetune/LSTM_Attn_tra/K10_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/info.json'),\n",
" 'K=20': glob.glob('output/search/finetune/LSTM_Attn_tra/K20_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/info.json')\n",
" \"K=1\": glob.glob(\"output/search/LSTM_Attn/hs256_bs1024_do0.1_lr0.0002_seed*/info.json\"),\n",
" \"K=3\": glob.glob(\n",
" \"output/search/finetune/LSTM_Attn_tra/K3_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/info.json\"\n",
" ),\n",
" \"K=5\": glob.glob(\n",
" \"output/search/finetune/LSTM_Attn_tra/K5_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/info.json\"\n",
" ),\n",
" \"K=10\": glob.glob(\n",
" \"output/search/finetune/LSTM_Attn_tra/K10_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/info.json\"\n",
" ),\n",
" \"K=20\": glob.glob(\n",
" \"output/search/finetune/LSTM_Attn_tra/K20_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/info.json\"\n",
" ),\n",
"}"
]
},
Expand All @@ -649,16 +683,11 @@
"source": [
"report = dict()\n",
"for k, v in exps.items():\n",
" \n",
" tmp = dict()\n",
" for fname in v:\n",
" with open(fname) as f:\n",
" info = json.load(f)\n",
" tmp[fname] = (\n",
" {\n",
" \"IC\":info[\"metric\"][\"IC\"],\n",
" \"MSE\":info[\"metric\"][\"MSE\"]\n",
" })\n",
" tmp[fname] = {\"IC\": info[\"metric\"][\"IC\"], \"MSE\": info[\"metric\"][\"MSE\"]}\n",
" tmp = pd.DataFrame(tmp).T\n",
" report[k] = tmp.mean()\n",
"report = pd.DataFrame(report).T"
Expand All @@ -681,13 +710,14 @@
}
],
"source": [
"fig, axes = plt.subplots(1, 2, figsize=(6,3)); axes = axes.flatten()\n",
"report['IC'].plot.bar(rot=30, ax=axes[0])\n",
"fig, axes = plt.subplots(1, 2, figsize=(6, 3))\n",
"axes = axes.flatten()\n",
"report[\"IC\"].plot.bar(rot=30, ax=axes[0])\n",
"axes[0].set_ylim(0.045, 0.062)\n",
"axes[0].set_title('IC performance')\n",
"report['MSE'].astype(float).plot.bar(rot=30, ax=axes[1], color='green')\n",
"axes[0].set_title(\"IC performance\")\n",
"report[\"MSE\"].astype(float).plot.bar(rot=30, ax=axes[1], color=\"green\")\n",
"axes[1].set_ylim(0.155, 0.1585)\n",
"axes[1].set_title('MSE performance')\n",
"axes[1].set_title(\"MSE performance\")\n",
"plt.tight_layout()\n",
"# plt.savefig('sensitivity.pdf')"
]
Expand Down
Loading