diff --git a/appveyor.yml b/appveyor.yml index 38da8ffd..387a1694 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -28,7 +28,7 @@ build: false # TODO modify this phase as you see fit test_script: - cargo build --verbose - - cargo test + - cargo test -j 1 before_deploy: # Generate artifacts for release diff --git a/bin/benchmark b/bin/benchmark index 6d50e965..ac4f0496 100755 --- a/bin/benchmark +++ b/bin/benchmark @@ -25,14 +25,15 @@ set -e -GAMES=500 - if [ $1 == "9" ]; then - TIME="5m" + TIME="2m" + GAMES=500 elif [ $1 == "13" ]; then TIME="10m" + GAMES=1000 elif [ $1 == "19" ]; then TIME="20m" + GAMES=1000 else echo "Size '$1' isn't supported!" exit 1 diff --git a/bin/benchmark-ec2 b/bin/benchmark-ec2 index cd180924..748c104f 100755 --- a/bin/benchmark-ec2 +++ b/bin/benchmark-ec2 @@ -28,14 +28,15 @@ set -e -GAMES=500 - if [ $1 == "9" ]; then - TIME="5m" + TIME="2m" + GAMES=500 elif [ $1 == "13" ]; then TIME="10m" + GAMES=1000 elif [ $1 == "19" ]; then TIME="20m" + GAMES=1000 else echo "Size '$1' isn't supported!" exit 1 diff --git a/bin/wins-from-benchmark-results.rb b/bin/wins-from-benchmark-results.rb index 0f87a3c7..e5254ead 100644 --- a/bin/wins-from-benchmark-results.rb +++ b/bin/wins-from-benchmark-results.rb @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # # Copyright (c) 2015 Urban Hafner +# Copyright (c) 2016 Urban Hafner # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation files @@ -21,22 +22,36 @@ # ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +require 'csv' -def data(contents) - contents.each_line.find_all {|l| l !~ /^#/ }.map do |l| - l.split(/\s+/)[3] - end +def parse_file(fn) + contents = File.read(fn) + relevant_lines = contents.each_line.find_all {|l| l !~ /^#/ } + CSV.parse(relevant_lines.map(&:strip).join("\n"), col_sep: "\t") end -def wins(file) - contents = File.read(file) - white = data(contents).find_all {|l| l =~ /W\+/ }.count - black = data(contents).find_all {|l| l =~ /B\+/ }.count +RES_B = 1 +RES_W = 2 +RES_R = 3 + +def wins(fn) + data = parse_file(fn) + white = data.find_all {|row| row[RES_R] =~ /W\+/ }.count + black = data.find_all {|row| row[RES_R] =~ /B\+/ }.count n = white + black p = white.to_f/n "#{(p*100).round(2)}% wins (#{white} games of #{n}, ± #{error(p: p, n: n, confidence: 0.95).round(2)} at 95%, ± #{error(p: p, n: n, confidence: 0.99).round(2)} at 99%)" end +def scoring(fn) + data = parse_file(fn) + relevant = data.find_all {|row| row[RES_R] !~ /[BW]\+R/ } + agreeing = relevant.find_all {|row| row[RES_W] == row[RES_B] }.count + n = relevant.length + p = agreeing.to_f/n + "#{(p*100).round(2)}% same score as GnuGo (#{agreeing} of #{n}, ± #{error(p: p, n: n, confidence: 0.95).round(2)} at 95%, ± #{error(p: p, n: n, confidence: 0.99).round(2)} at 99%)" +end + def z(confidence:) alpha = 1 - confidence (1 - 0.5*alpha)*2 @@ -48,5 +63,7 @@ def error(p:, n:, confidence:) Dir["*.dat"].each do |fn| next if fn =~ /summary\.dat/ - puts "#{fn}: #{wins(fn)}" + puts "#{fn}:" + puts "\t\t#{wins(fn)}" + puts "\t\t#{scoring(fn)}" end diff --git a/ci/script.sh b/ci/script.sh index 3a099a26..96657b1f 100644 --- a/ci/script.sh +++ b/ci/script.sh @@ -39,7 +39,7 @@ run_test_suite() { fi cargo build --target $TARGET --verbose - cargo test --target $TARGET + cargo test --target $TARGET -j 1 } main() { diff --git a/fixtures/sgf/no-legal-moves-left.sgf b/fixtures/sgf/no-legal-moves-left.sgf new file mode 100644 index 00000000..2e0d422e --- /dev/null +++ b/fixtures/sgf/no-legal-moves-left.sgf @@ -0,0 +1,3 @@ +(;FF[4]CA[UTF-8]AP[GoGui:1.4.9]SZ[3] +KM[0]DT[2016-05-18] +;B[ba];W[];B[bb];W[];B[bc];W[];B[ab];W[];B[cb]) diff --git a/src/board/coord/mod.rs b/src/board/coord/mod.rs index f1cc811b..40f3f79f 100644 --- a/src/board/coord/mod.rs +++ b/src/board/coord/mod.rs @@ -2,6 +2,7 @@ * * * Copyright 2014 Urban Hafner, Thomas Poinsot * * Copyright 2015 Urban Hafner, Igor Polyakov * + * Copyright 2016 Urban Hafner * * * * This file is part of Iomrascálaí. * * * @@ -20,7 +21,6 @@ * * ************************************************************************/ use core::fmt; -use std::cmp::Eq; mod test; diff --git a/src/config/defaults.toml b/src/config/defaults.toml index ede32d81..2772bced 100644 --- a/src/config/defaults.toml +++ b/src/config/defaults.toml @@ -34,3 +34,4 @@ score_weight = 0.0653414 ownership_prior = 87 ownership_cutoff = 0.892867 +playouts = 10000 diff --git a/src/config/mod.rs b/src/config/mod.rs index ec2c7d24..fd8440a4 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -286,6 +286,8 @@ pub struct ScoringConfig { /// Value between 0.0 and 1.0 which is the cutoff above which a /// point is considered to be owned by a color. pub ownership_cutoff: f32, + /// Number of playouts to run when trying to determine the final score of a board. + pub playouts: usize, } impl ScoringConfig { @@ -299,6 +301,7 @@ impl ScoringConfig { ScoringConfig { ownership_prior: Self::as_integer(&table, "ownership_prior"), ownership_cutoff: Self::as_float(&table, "ownership_cutoff"), + playouts: Self::as_integer(&table, "playouts"), } } @@ -351,7 +354,9 @@ impl Config { #[test] pub fn test_config() -> Config { - Self::default(false, false, Ruleset::KgsChinese, Some(1)) + let mut config = Self::default(false, false, Ruleset::KgsChinese, Some(1)); + config.scoring.playouts = 10; + config } /// Uses the TOML returned by `Config::toml()` and returns a diff --git a/src/engine/controller/mod.rs b/src/engine/controller/mod.rs index eb3c1e16..53459caa 100644 --- a/src/engine/controller/mod.rs +++ b/src/engine/controller/mod.rs @@ -34,6 +34,7 @@ use std::sync::Arc; pub struct EngineController { config: Arc, engine: Engine, + run_playouts_for_scoring: bool, } impl EngineController { @@ -42,10 +43,12 @@ impl EngineController { EngineController { config: config, engine: engine, + run_playouts_for_scoring: true, } } pub fn reset(&mut self, size: u8, komi: f32) { + self.run_playouts_for_scoring = true; self.engine.reset(size, komi); } @@ -53,20 +56,28 @@ impl EngineController { format!("{}", self.ownership()) } - pub fn final_score(&self, game: &Game) -> String { + pub fn final_score(&mut self, game: &Game) -> String { + self.run_playouts(game); FinalScore::new(self.config.clone(), game, self.ownership()).score() } - pub fn final_status_list(&self, game: &Game, kind: &str) -> Result { + pub fn final_status_list(&mut self, game: &Game, kind: &str) -> Result { + self.run_playouts(game); FinalScore::new(self.config.clone(), game, self.ownership()).status_list(kind) + } + pub fn donplayouts(&mut self, game: &Game, playouts: usize) { + self.run_playouts_for_scoring = false; + self.engine.donplayouts(game, playouts); } pub fn genmove(&mut self, color: Color, game: &Game, timer: &Timer) -> (Move, usize) { + self.run_playouts_for_scoring = true; self.engine.genmove(color, game, timer) } pub fn genmove_cleanup(&mut self, color: Color, game: &Game, timer: &Timer) -> (Move, usize) { + self.run_playouts_for_scoring = true; self.engine.genmove_cleanup(color, game, timer) } @@ -74,4 +85,9 @@ impl EngineController { &self.engine.ownership() } + fn run_playouts(&mut self, game: &Game) { + if !self.run_playouts_for_scoring { return; } + let playouts = self.config.scoring.playouts; + self.donplayouts(game, playouts); + } } diff --git a/src/engine/mod.rs b/src/engine/mod.rs index 16b8fff9..0bba51b5 100644 --- a/src/engine/mod.rs +++ b/src/engine/mod.rs @@ -28,6 +28,7 @@ use board::Move; use board::NoMove; use board::Pass; use board::Resign; +use board::White; use config::Config; use game::Game; use ownership::OwnershipStatistics; @@ -150,14 +151,28 @@ impl Engine { self.config.log(format!("No moves to simulate!")); return (Pass(color), self.root.playouts()); } + let stop = |win_ratio, _| { timer.ran_out_of_time(win_ratio) }; + self.search(game, stop); + let msg = format!("{} simulations ({}% wins on average, {} nodes)", self.root.playouts(), self.root.win_ratio()*100.0, self.root.descendants()); + self.config.log(msg); + let playouts = self.root.playouts(); + let m = self.best_move(game, color, cleanup); + self.set_new_root(&game.play(m).unwrap(), color); + (m,playouts) + } + + fn search(&mut self, game: &Game, stop: F) where F: Fn(f32, usize) -> bool { self.spin_up(game); loop { let win_ratio = { let (best, _) = self.root.best(); best.win_ratio() }; - if timer.ran_out_of_time(win_ratio) { - return self.finish(game, color, cleanup); + let done = { + stop(win_ratio, self.root.playouts()) + }; + if done { + return self.spin_down(); } let r = self.receive_from_threads.recv(); check!(self.config, res = r => { @@ -166,6 +181,22 @@ impl Engine { } } + pub fn donplayouts(&mut self, game: &Game, playouts: usize) { + self.ownership = OwnershipStatistics::new(self.config.clone(), game.size(), game.komi()); + if self.root.has_no_children() { + let color = match game.last_move() { + NoMove => White, + _ => *game.last_move().color() + }; + self.root = Node::root(game, color, self.config.clone()); + } + let initial_playouts = self.root.playouts(); + let stop = |_, current_playouts: usize| { + (current_playouts - initial_playouts) > playouts + }; + self.search(game, stop); + } + fn dead_stones_on_board(&self, game: &Game) -> bool { FinalScore::new(self.config.clone(), game, self.ownership()).dead_stones_on_board() } @@ -283,18 +314,12 @@ impl Engine { } } - fn finish(&mut self, game: &Game, color: Color, cleanup: bool) -> (Move,usize) { + fn spin_down(&mut self) { self.id += 1; for halt_sender in &self.halt_senders { check!(self.config, halt_sender.send(())); } self.halt_senders = vec!(); - let msg = format!("{} simulations ({}% wins on average, {} nodes)", self.root.playouts(), self.root.win_ratio()*100.0, self.root.descendants()); - self.config.log(msg); - let playouts = self.root.playouts(); - let m = self.best_move(game, color, cleanup); - self.set_new_root(&game.play(m).unwrap(), color); - (m,playouts) } fn spin_up(&mut self, game: &Game) { diff --git a/src/gtp/mod.rs b/src/gtp/mod.rs index 03f64162..20c13125 100644 --- a/src/gtp/mod.rs +++ b/src/gtp/mod.rs @@ -63,6 +63,7 @@ impl<'a> GTPInterpreter<'a> { "final_status_list", "genmove", "gogui-analyze_commands", + "imrscl-donplayouts", "imrscl-ownership", "kgs-genmove_cleanup", "known_command", @@ -125,6 +126,7 @@ impl<'a> GTPInterpreter<'a> { "final_status_list" => self.execute_final_status_list(arguments), "genmove" => self.execute_genmove(arguments), "gogui-analyze_commands" => self.execute_gogui_analyze_commands(arguments), + "imrscl-donplayouts" => self.execute_imrscl_donplayouts(arguments), "imrscl-ownership" => self.execute_imrscl_ownership(arguments), "kgs-genmove_cleanup" => self.execute_kgs_genmove_cleanup(arguments), "known_command" => self.execute_known_command(arguments), @@ -276,6 +278,21 @@ impl<'a> GTPInterpreter<'a> { } } + fn execute_imrscl_donplayouts(&mut self, arguments: &[&str]) -> Result { + match arguments.get(0) { + Some(playouts_str) => { + match playouts_str.parse() { + Ok(playouts) => { + self.controller.donplayouts(&self.game, playouts); + Ok("".to_string()) + }, + Err(e) => Err(format!("{:?}", e)) + } + } + None => Err("missing argument".to_string()), + } + } + fn execute_imrscl_ownership(&mut self, _: &[&str]) -> Result { let stats = self.controller.ownership_statistics(); Ok(stats) @@ -307,12 +324,14 @@ impl<'a> GTPInterpreter<'a> { } fn execute_final_score(&mut self, _: &[&str]) -> Result { + self.game.reset_game_over(); Ok(self.controller.final_score(&self.game)) } fn execute_final_status_list(&mut self, arguments: &[&str]) -> Result { match arguments.get(0) { Some(kind) => { + self.game.reset_game_over(); self.controller.final_status_list(&self.game, kind) }, None => Err("missing argument".to_string()) diff --git a/src/gtp/test.rs b/src/gtp/test.rs index 30f57b63..4b443e9e 100644 --- a/src/gtp/test.rs +++ b/src/gtp/test.rs @@ -33,6 +33,7 @@ pub use super::GTPInterpreter; pub use hamcrest::assert_that; pub use hamcrest::equal_to; pub use hamcrest::is; +pub use hamcrest::is_not; pub use std::sync::Arc; pub fn err(s: &'static str) -> Result { @@ -141,6 +142,14 @@ describe! interpreter { let response = interpreter.read("genmove b\n"); assert!(response.is_ok()); } + + it "does not generate a pass on an empty board" { + interpreter.read("boardsize 9\n").unwrap(); + interpreter.read("clear_board\n").unwrap(); + let response = interpreter.read("genmove b\n"); + assert!(response.is_ok()); + assert_that(response.unwrap(), is_not(equal_to("pass".to_string()))); + } } describe! kgs { @@ -229,7 +238,7 @@ describe! interpreter { it "no newline at end" { let response = interpreter.read("list_commands\n"); - let expected = "boardsize\nclear_board\nfinal_score\nfinal_status_list\ngenmove\ngogui-analyze_commands\nimrscl-ownership\nkgs-genmove_cleanup\nknown_command\nkomi\nlist_commands\nloadsgf\nname\nplay\nprotocol_version\nquit\nreg_genmove\nshowboard\ntime_left\ntime_settings\nversion"; + let expected = "boardsize\nclear_board\nfinal_score\nfinal_status_list\ngenmove\ngogui-analyze_commands\nimrscl-donplayouts\nimrscl-ownership\nkgs-genmove_cleanup\nknown_command\nkomi\nlist_commands\nloadsgf\nname\nplay\nprotocol_version\nquit\nreg_genmove\nshowboard\ntime_left\ntime_settings\nversion"; assert_that(response, is(equal_to(ok(expected)))); } @@ -275,6 +284,26 @@ describe! interpreter { assert_that(response, is(equal_to(ok("B+9.5")))); } + it "doesn't crash after loading a SGF file" { + interpreter.read("loadsgf fixtures/sgf/twomoves.sgf\n").unwrap(); + let response = interpreter.read("final_score\n"); + assert!(response.is_ok()); + } + + it "doesn't crash after loading a completed game" { + interpreter.read("boardsize 9\n").unwrap(); + interpreter.read("clear_board\n").unwrap(); + interpreter.read("play b pass\n").unwrap(); + interpreter.read("play w pass\n").unwrap(); + let response = interpreter.read("final_score\n"); + assert_that(response, is(equal_to(ok("W+6.5")))); + } + + it "doesn't crash after loading a game with no legal moves" { + interpreter.read("loadsgf fixtures/sgf/no-legal-moves-left.sgf\n").unwrap(); + let response = interpreter.read("final_score\n"); + assert_that(response, is(equal_to(ok("B+9")))); + } } describe! name { @@ -326,10 +355,10 @@ describe! interpreter { describe! final_status_list { before_each { - interpreter.read("boardsize 3\n").unwrap(); + interpreter.read("boardsize 9\n").unwrap(); interpreter.read("clear_board\n").unwrap(); interpreter.read("play b a1\n").unwrap(); - interpreter.read("play w b2\n").unwrap(); + interpreter.read("play w b9\n").unwrap(); } it "reports no dead stones" { @@ -337,9 +366,9 @@ describe! interpreter { assert_that(response, is(equal_to(ok("")))); } - it "reports one alive stone" { + it "reports two alive stones" { let response = interpreter.read("final_status_list alive\n"); - assert_that(response, is(equal_to(ok("A1 B2")))); + assert_that(response, is(equal_to(ok("A1 B9")))); } it "reports no seki stones" { @@ -356,6 +385,28 @@ describe! interpreter { let response = interpreter.read("final_status_list\n"); assert_that(response, is(equal_to(err("missing argument")))); } + + it "doesn't crash after loading a SGF file" { + interpreter.read("loadsgf fixtures/sgf/twomoves.sgf\n").unwrap(); + let response = interpreter.read("final_status_list dead\n"); + assert!(response.is_ok()); + } + + it "doesn't crash after loading a completed game" { + interpreter.read("boardsize 9\n").unwrap(); + interpreter.read("clear_board\n").unwrap(); + interpreter.read("play b pass\n").unwrap(); + interpreter.read("play w pass\n").unwrap(); + let response = interpreter.read("final_status_list dead\n"); + assert_that(response, is(equal_to(ok("")))); + } + + it "doesn't crash after loading a game with no legal moves" { + interpreter.read("loadsgf fixtures/sgf/no-legal-moves-left.sgf\n").unwrap(); + let response = interpreter.read("final_status_list dead\n"); + assert_that(response, is(equal_to(ok("")))); + } + } // Gogui extensions @@ -424,10 +475,10 @@ describe! interpreter { describe! final_status_list { before_each { - interpreter.read("boardsize 3\n").unwrap(); + interpreter.read("boardsize 9\n").unwrap(); interpreter.read("clear_board\n").unwrap(); interpreter.read("play b a1\n").unwrap(); - interpreter.read("play w b2\n").unwrap(); + interpreter.read("play w b9\n").unwrap(); } it "reports no dead stones" { @@ -435,9 +486,9 @@ describe! interpreter { assert_that(response, is(equal_to(ok("")))); } - it "reports one alive stone" { + it "reports two alive stone" { let response = interpreter.read("final_status_list alive\n"); - assert_that(response, is(equal_to(ok("A1 B2")))); + assert_that(response, is(equal_to(ok("A1 B9")))); } it "reports no seki stones" { diff --git a/src/ownership/mod.rs b/src/ownership/mod.rs index e7d786c4..feba39cb 100644 --- a/src/ownership/mod.rs +++ b/src/ownership/mod.rs @@ -1,6 +1,7 @@ /************************************************************************ * * * Copyright 2015 Urban Hafner * + * Copyright 2016 Urban Hafner * * * * This file is part of Iomrascálaí. * * * @@ -79,10 +80,7 @@ impl OwnershipStatistics { let index = coord.to_index(self.size); let b = self.black[index]; let w = self.white[index]; - let e = self.empty[index]; - let count = b + w + e; - let fraction = cmp::max(b,w) as f32 / count as f32; - if fraction > self.config.scoring.ownership_cutoff { + if self.coord_decided(coord) { if b > w { Black } else { @@ -93,6 +91,16 @@ impl OwnershipStatistics { } } + fn coord_decided(&self, coord: &Coord) -> bool { + let index = coord.to_index(self.size); + let b = self.black[index]; + let w = self.white[index]; + let e = self.empty[index]; + let count = b + w + e; + let fraction = cmp::max(b,w) as f32 / count as f32; + fraction > self.config.scoring.ownership_cutoff + } + pub fn gfx(&self) -> String { let mut b = String::from("BLACK"); let mut w = String::from("WHITE");