From 407f308cc543286e39701caf0acd1afab39dde2c Mon Sep 17 00:00:00 2001
From: Peilin Yang <yangpeilyn@gmail.com>
Date: Wed, 2 Jan 2019 06:34:57 -0800
Subject: [PATCH] Added fine tuning results for axiomatic reranking (#542)

---
 src/main/python/fine_tuning/README.md      |  5 +++
 src/main/resources/fine_tuning/models.yaml | 50 +++++++++++-----------
 2 files changed, 30 insertions(+), 25 deletions(-)

diff --git a/src/main/python/fine_tuning/README.md b/src/main/python/fine_tuning/README.md
index d379db5bdd..d0151e8374 100644
--- a/src/main/python/fine_tuning/README.md
+++ b/src/main/python/fine_tuning/README.md
@@ -23,4 +23,9 @@ python src/main/python/fine_tuning/run_batch.py --collection robust04 --basemode
 python src/main/python/fine_tuning/run_batch.py --collection robust04 --model bm25+rm3 --n 44 --run --use_drr_fold
 ```
 
+##### BM25+AxiomaticReranking Robust04 (runs + eval + print results)
+```
+python src/main/python/fine_tuning/run_batch.py --collection robust04 --model bm25+axiom --n 44 --run --use_drr_fold
+```
+
 ##### Change `robust04` to `cw0b9` for all ClueWeb09b results
diff --git a/src/main/resources/fine_tuning/models.yaml b/src/main/resources/fine_tuning/models.yaml
index c8c61ca44d..dbed4554de 100644
--- a/src/main/resources/fine_tuning/models.yaml
+++ b/src/main/resources/fine_tuning/models.yaml
@@ -79,20 +79,20 @@ models:
     expected:
       robust04:
         map:
-          best_avg: 0.2955
-          oracles_per_topic: 0.4044
-          2-fold: 0.2929
-          5-fold: 0.2878
+          best_avg: 0.2967
+          oracles_per_topic: 0.4024
+          2-fold: 0.2897
+          5-fold: 0.2914
         P_20:
-          best_avg: 0.3932
-          oracles_per_topic: 0.5661
-          2-fold: 0.3883
-          5-fold: 0.3902
+          best_avg: 0.3924
+          oracles_per_topic: 0.5651
+          2-fold: 0.3877
+          5-fold: 0.3854
         ndcg20:
-          best_avg: 0.44484
-          oracles_per_topic: 0.6278
-          2-fold: 0.4361
-          5-fold: 0.4375
+          best_avg: 0.44386
+          oracles_per_topic: 0.6262
+          2-fold: 0.4371
+          5-fold: 0.4326
   bm25+axiom:
     name: axiom
     fixed_params: "-inmem -skipexists -axiom.n 30 -axiom.deterministic -rerankCutoff 50"
@@ -106,7 +106,7 @@ models:
         type: float
         lower: 0
         upper: 0.5
-        pace: 0.05
+        pace: 0.1
       axiom.r:
         type: int
         lower: 1
@@ -125,20 +125,20 @@ models:
     expected:
       robust04:
         map:
-          best_avg: 0.0
-          oracles_per_topic: 0.0
-          2-fold: 0.0
-          5-fold: 0.0
+          best_avg: 0.2983
+          oracles_per_topic: 0.4168
+          2-fold: 0.2933
+          5-fold: 0.2920
         P_20:
-          best_avg: 0.0
-          oracles_per_topic: 0.0
-          2-fold: 0.0
-          5-fold: 0.0
+          best_avg: 0.3932
+          oracles_per_topic: 0.5841
+          2-fold: 0.3821
+          5-fold: 0.3856
         ndcg20:
-          best_avg: 0.0
-          oracles_per_topic: 0.0
-          2-fold: 0.0
-          5-fold: 0.0
+          best_avg: 0.44498
+          oracles_per_topic: 0.64720
+          2-fold: 0.4282
+          5-fold: 0.4302
   rm3:
     name: rm3
     fixed_params: "-inmem -skipexists -rerankCutoff 50"