Fix scoring

appveyor.yml

@@ -28,7 +28,7 @@ build: false
 # TODO modify this phase as you see fit
 test_script:
   - cargo build --verbose
-  - cargo test
+  - cargo test -j 1
 
 before_deploy:
   # Generate artifacts for release

bin/benchmark

@@ -25,14 +25,15 @@
 
 set -e
 
-GAMES=500
-
 if [ $1 == "9" ]; then
-    TIME="5m"
+    TIME="2m"
+    GAMES=500
 elif [ $1 == "13" ]; then
     TIME="10m"
+    GAMES=1000
 elif [ $1 == "19" ]; then
     TIME="20m"
+    GAMES=1000
 else
     echo "Size '$1' isn't supported!"
     exit 1

bin/benchmark-ec2

@@ -28,14 +28,15 @@
 
 set -e
 
-GAMES=500
-
 if [ $1 == "9" ]; then
-    TIME="5m"
+    TIME="2m"
+    GAMES=500
 elif [ $1 == "13" ]; then
     TIME="10m"
+    GAMES=1000
 elif [ $1 == "19" ]; then
     TIME="20m"
+    GAMES=1000
 else
     echo "Size '$1' isn't supported!"
     exit 1

bin/wins-from-benchmark-results.rb

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # Copyright (c) 2015 Urban Hafner
+# Copyright (c) 2016 Urban Hafner
 #
 # Permission is hereby granted, free of charge, to any person
 # obtaining a copy of this software and associated documentation files
@@ -21,22 +22,36 @@
 # ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
+require 'csv'
 
-def data(contents)
-  contents.each_line.find_all {|l| l !~ /^#/ }.map do |l|
-    l.split(/\s+/)[3]
-  end
+def parse_file(fn)
+  contents = File.read(fn)
+  relevant_lines = contents.each_line.find_all {|l| l !~ /^#/ }
+  CSV.parse(relevant_lines.map(&:strip).join("\n"), col_sep: "\t")
 end
 
-def wins(file)
-  contents = File.read(file)
-  white = data(contents).find_all {|l| l =~ /W\+/ }.count
-  black = data(contents).find_all {|l| l =~ /B\+/ }.count
+RES_B = 1
+RES_W = 2
+RES_R = 3
+
+def wins(fn)
+  data = parse_file(fn)
+  white = data.find_all {|row| row[RES_R] =~ /W\+/ }.count
+  black = data.find_all {|row| row[RES_R] =~ /B\+/ }.count
   n = white + black
   p = white.to_f/n
   "#{(p*100).round(2)}% wins (#{white} games of #{n}, ± #{error(p: p, n: n, confidence: 0.95).round(2)} at 95%, ± #{error(p: p, n: n, confidence: 0.99).round(2)} at 99%)"
 end
 
+def scoring(fn)
+  data = parse_file(fn)
+  relevant = data.find_all {|row| row[RES_R] !~ /[BW]\+R/ }
+  agreeing = relevant.find_all {|row| row[RES_W] == row[RES_B] }.count
+  n = relevant.length
+  p = agreeing.to_f/n
+  "#{(p*100).round(2)}% same score as GnuGo (#{agreeing} of #{n}, ± #{error(p: p, n: n, confidence: 0.95).round(2)} at 95%, ± #{error(p: p, n: n, confidence: 0.99).round(2)} at 99%)"
+end
+
 def z(confidence:)
   alpha = 1 - confidence
   (1 - 0.5*alpha)*2
@@ -48,5 +63,7 @@ def error(p:, n:, confidence:)
 
 Dir["*.dat"].each do |fn|
   next if fn =~ /summary\.dat/
-  puts "#{fn}: #{wins(fn)}"
+  puts "#{fn}:"
+  puts "\t\t#{wins(fn)}"
+  puts "\t\t#{scoring(fn)}"
 end

ci/script.sh

@@ -39,7 +39,7 @@ run_test_suite() {
     fi
 
     cargo build --target $TARGET --verbose
-    cargo test --target $TARGET
+    cargo test --target $TARGET -j 1
 }
 
 main() {

fixtures/sgf/no-legal-moves-left.sgf

@@ -0,0 +1,3 @@
+(;FF[4]CA[UTF-8]AP[GoGui:1.4.9]SZ[3]
+KM[0]DT[2016-05-18]
+;B[ba];W[];B[bb];W[];B[bc];W[];B[ab];W[];B[cb])

src/board/coord/mod.rs

@@ -2,6 +2,7 @@
  *                                                                      *
  * Copyright 2014 Urban Hafner, Thomas Poinsot                          *
  * Copyright 2015 Urban Hafner, Igor Polyakov                           *
+ * Copyright 2016 Urban Hafner                                          *
  *                                                                      *
  * This file is part of Iomrascálaí.                                    *
  *                                                                      *
@@ -20,7 +21,6 @@
  *                                                                      *
  ************************************************************************/
 use core::fmt;
-use std::cmp::Eq;
 
 mod test;
 

src/config/defaults.toml

@@ -34,3 +34,4 @@ score_weight = 0.0653414
 
 ownership_prior = 87
 ownership_cutoff = 0.892867
+playouts = 10000

src/config/mod.rs

@@ -286,6 +286,8 @@ pub struct ScoringConfig {
     /// Value between 0.0 and 1.0 which is the cutoff above which a
     /// point is considered to be owned by a color.
     pub ownership_cutoff: f32,
+    /// Number of playouts to run when trying to determine the final score of a board.
+    pub playouts: usize,
 }
 
 impl ScoringConfig {
@@ -299,6 +301,7 @@ impl ScoringConfig {
         ScoringConfig {
             ownership_prior: Self::as_integer(&table, "ownership_prior"),
             ownership_cutoff: Self::as_float(&table, "ownership_cutoff"),
+            playouts: Self::as_integer(&table, "playouts"),
         }
     }
 
@@ -351,7 +354,9 @@ impl Config {
 
     #[test]
     pub fn test_config() -> Config {
-        Self::default(false, false, Ruleset::KgsChinese, Some(1))
+        let mut config = Self::default(false, false, Ruleset::KgsChinese, Some(1));
+        config.scoring.playouts = 10;
+        config
     }
 
     /// Uses the TOML returned by `Config::toml()` and returns a

src/engine/controller/mod.rs

@@ -34,6 +34,7 @@ use std::sync::Arc;
 pub struct EngineController {
     config: Arc<Config>,
     engine: Engine,
+    run_playouts_for_scoring: bool,
 }
 
 impl EngineController {
@@ -42,36 +43,51 @@ impl EngineController {
         EngineController {
             config: config,
             engine: engine,
+            run_playouts_for_scoring: true,
         }
     }
 
     pub fn reset(&mut self, size: u8, komi: f32) {
+        self.run_playouts_for_scoring = true;
         self.engine.reset(size, komi);
     }
 
     pub fn ownership_statistics(&self) -> String {
         format!("{}", self.ownership())
     }
 
-    pub fn final_score(&self, game: &Game) -> String {
+    pub fn final_score(&mut self, game: &Game) -> String {
+        self.run_playouts(game);
         FinalScore::new(self.config.clone(), game, self.ownership()).score()
     }
 
-    pub fn final_status_list(&self, game: &Game, kind: &str) -> Result<String, String> {
+    pub fn final_status_list(&mut self, game: &Game, kind: &str) -> Result<String, String> {
+        self.run_playouts(game);
         FinalScore::new(self.config.clone(), game, self.ownership()).status_list(kind)
+    }
 
+    pub fn donplayouts(&mut self, game: &Game, playouts: usize) {
+        self.run_playouts_for_scoring = false;
+        self.engine.donplayouts(game, playouts);
     }
 
     pub fn genmove(&mut self, color: Color, game: &Game, timer: &Timer) -> (Move, usize) {
+        self.run_playouts_for_scoring = true;
         self.engine.genmove(color, game, timer)
     }
 
     pub fn genmove_cleanup(&mut self, color: Color, game: &Game, timer: &Timer) -> (Move, usize) {
+        self.run_playouts_for_scoring = true;
         self.engine.genmove_cleanup(color, game, timer)
     }
 
     fn ownership(&self) -> &OwnershipStatistics {
         &self.engine.ownership()
     }
 
+    fn run_playouts(&mut self, game: &Game) {
+        if !self.run_playouts_for_scoring { return; }
+        let playouts = self.config.scoring.playouts;
+        self.donplayouts(game, playouts);
+    }
 }

src/engine/mod.rs

@@ -28,6 +28,7 @@ use board::Move;
 use board::NoMove;
 use board::Pass;
 use board::Resign;
+use board::White;
 use config::Config;
 use game::Game;
 use ownership::OwnershipStatistics;
@@ -150,14 +151,28 @@ impl Engine {
             self.config.log(format!("No moves to simulate!"));
             return (Pass(color), self.root.playouts());
         }
+        let stop = |win_ratio, _| { timer.ran_out_of_time(win_ratio) };
+        self.search(game, stop);
+        let msg = format!("{} simulations ({}% wins on average, {} nodes)", self.root.playouts(), self.root.win_ratio()*100.0, self.root.descendants());
+        self.config.log(msg);
+        let playouts = self.root.playouts();
+        let m = self.best_move(game, color, cleanup);
+        self.set_new_root(&game.play(m).unwrap(), color);
+        (m,playouts)
+    }
+
+    fn search<F>(&mut self, game: &Game, stop: F) where F: Fn(f32, usize) -> bool {
         self.spin_up(game);
         loop {
             let win_ratio = {
                 let (best, _) = self.root.best();
                 best.win_ratio()
             };
-            if timer.ran_out_of_time(win_ratio) {
-                return self.finish(game, color, cleanup);
+            let done = {
+                stop(win_ratio, self.root.playouts())
+            };
+            if done {
+                return self.spin_down();
             }
             let r = self.receive_from_threads.recv();
             check!(self.config, res = r => {
@@ -166,6 +181,22 @@ impl Engine {
         }
     }
 
+    pub fn donplayouts(&mut self, game: &Game, playouts: usize) {
+        self.ownership = OwnershipStatistics::new(self.config.clone(), game.size(), game.komi());
+        if self.root.has_no_children() {
+            let color = match game.last_move() {
+                NoMove => White,
+                _ => *game.last_move().color()
+            };
+            self.root = Node::root(game, color, self.config.clone());
+        }
+        let initial_playouts = self.root.playouts();
+        let stop = |_, current_playouts: usize| {
+            (current_playouts - initial_playouts) > playouts
+        };
+        self.search(game, stop);
+    }
+
     fn dead_stones_on_board(&self, game: &Game) -> bool {
         FinalScore::new(self.config.clone(), game, self.ownership()).dead_stones_on_board()
     }
@@ -283,18 +314,12 @@ impl Engine {
         }
     }
 
-    fn finish(&mut self, game: &Game, color: Color, cleanup: bool) -> (Move,usize) {
+    fn spin_down(&mut self) {
         self.id += 1;
         for halt_sender in &self.halt_senders {
             check!(self.config, halt_sender.send(()));
         }
         self.halt_senders = vec!();
-        let msg = format!("{} simulations ({}% wins on average, {} nodes)", self.root.playouts(), self.root.win_ratio()*100.0, self.root.descendants());
-        self.config.log(msg);
-        let playouts = self.root.playouts();
-        let m = self.best_move(game, color, cleanup);
-        self.set_new_root(&game.play(m).unwrap(), color);
-        (m,playouts)
     }
 
     fn spin_up(&mut self, game: &Game) {

src/gtp/mod.rs

@@ -63,6 +63,7 @@ impl<'a> GTPInterpreter<'a> {
             "final_status_list",
             "genmove",
             "gogui-analyze_commands",
+            "imrscl-donplayouts",
             "imrscl-ownership",
             "kgs-genmove_cleanup",
             "known_command",
@@ -125,6 +126,7 @@ impl<'a> GTPInterpreter<'a> {
             "final_status_list" => self.execute_final_status_list(arguments),
             "genmove" => self.execute_genmove(arguments),
             "gogui-analyze_commands" => self.execute_gogui_analyze_commands(arguments),
+            "imrscl-donplayouts" => self.execute_imrscl_donplayouts(arguments),
             "imrscl-ownership" => self.execute_imrscl_ownership(arguments),
             "kgs-genmove_cleanup" => self.execute_kgs_genmove_cleanup(arguments),
             "known_command" => self.execute_known_command(arguments),
@@ -276,6 +278,21 @@ impl<'a> GTPInterpreter<'a> {
     	}
     }
 
+    fn execute_imrscl_donplayouts(&mut self, arguments: &[&str]) -> Result<String, String> {
+        match arguments.get(0) {
+            Some(playouts_str) => {
+                match playouts_str.parse() {
+                    Ok(playouts) => {
+                        self.controller.donplayouts(&self.game, playouts);
+                        Ok("".to_string())
+                    },
+                    Err(e) => Err(format!("{:?}", e))
+                }
+            }
+            None => Err("missing argument".to_string()),
+        }
+    }
+
     fn execute_imrscl_ownership(&mut self, _: &[&str]) -> Result<String, String> {
         let stats = self.controller.ownership_statistics();
         Ok(stats)
@@ -307,12 +324,14 @@ impl<'a> GTPInterpreter<'a> {
     }
 
     fn execute_final_score(&mut self, _: &[&str]) -> Result<String, String> {
+        self.game.reset_game_over();
         Ok(self.controller.final_score(&self.game))
     }
 
     fn execute_final_status_list(&mut self, arguments: &[&str]) -> Result<String, String> {
         match arguments.get(0) {
             Some(kind) => {
+                self.game.reset_game_over();
                 self.controller.final_status_list(&self.game, kind)
             },
             None => Err("missing argument".to_string())