From f941f5ac72d860f1f583392cbeb69d0694373824 Mon Sep 17 00:00:00 2001
From: Andrew Jackson <andrew.m.jackson@gmail.com>
Date: Mon, 5 Feb 2018 12:20:09 -0800
Subject: [PATCH] update selfplay to write 5% of games to a holdout directory
 (#57)

* update selfplay to write 5% of games to a holdout directory

* comments

Fixes #43
---
 main.py          | 15 ++++++++++++---
 preprocessing.py |  6 +++---
 rl_loop.py       |  4 +++-
 3 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/main.py b/main.py
index f663c8e6c..7855c61c1 100644
--- a/main.py
+++ b/main.py
@@ -118,10 +118,12 @@ def evaluate(
 def selfplay(
         load_file: "The path to the network model files",
         output_dir: "Where to write the games"="data/selfplay",
+        holdout_dir: "Where to write the games"="data/holdout",
         output_sgf: "Where to write the sgfs"="sgf/",
         readouts: 'How many simulations to run per move'=100,
         verbose: '>=2 will print debug info, >=3 will print boards' = 1,
-        resign_threshold: 'absolute value of threshold to resign at' = 0.95):
+        resign_threshold: 'absolute value of threshold to resign at' = 0.95
+        holdout_pct: 'how many games to hold out for evaluation' = 0.05):
     _ensure_dir_exists(output_sgf)
     _ensure_dir_exists(output_dir)
 
@@ -138,8 +140,15 @@ def selfplay(
     with gfile.GFile(os.path.join(output_sgf, '{}.sgf'.format(output_name)), 'w') as f:
         f.write(player.to_sgf())
 
-    fname = os.path.join(output_dir, "{}.tfrecord.zz".format(output_name))
-    preprocessing.make_dataset_from_selfplay(game_data, fname)
+    tf_examples = preprocessing.make_dataset_from_selfplay(game_data)
+
+    # Hold out 5% of games for evaluation.
+    if random.random() < holdout_pct:
+        fname = os.path.join(holdout_dir, "{}.tfrecord.zz".format(output_name))
+    else:
+        fname = os.path.join(output_dir, "{}.tfrecord.zz".format(output_name))
+
+    preprocessing.write_tf_examples(fname, tf_examples)
 
 
 def gather(
diff --git a/preprocessing.py b/preprocessing.py
index acfa18ede..6c1db3042 100644
--- a/preprocessing.py
+++ b/preprocessing.py
@@ -161,15 +161,15 @@ def get_input_tensors(batch_size, tf_records, num_repeats=None,
 # End-to-end utility functions
 
 
-def make_dataset_from_selfplay(data_extracts, tf_record):
+def make_dataset_from_selfplay(data_extracts):
     '''
+    Returns an iterable of tf.Examples.
     Args:
         data_extracts: An iterable of (position, pi, result) tuples
-        tf_record: name of file to write to
     '''
     tf_examples = (make_tf_example(features_lib.extract_features(pos), pi, result)
                    for pos, pi, result in data_extracts)
-    write_tf_examples(tf_record, tf_examples)
+    return tf_examples
 
 
 def make_dataset_from_sgf(sgf_filename, tf_record):
diff --git a/rl_loop.py b/rl_loop.py
index 6d1510987..f723dd8bf 100644
--- a/rl_loop.py
+++ b/rl_loop.py
@@ -27,7 +27,8 @@
 
 BASE_DIR = "gs://{}".format(BUCKET_NAME)
 MODELS_DIR = os.path.join(BASE_DIR, 'models')
-SELFPLAY_DIR = os.path.join(BASE_DIR, 'games')
+SELFPLAY_DIR = os.path.join(BASE_DIR, 'data/selfplay')
+HOLDOUT_DIR = os.path.join(BASE_DIR, 'data/holdout')
 SGF_DIR = os.path.join(BASE_DIR, 'sgf')
 TRAINING_CHUNK_DIR = os.path.join(BASE_DIR, 'data', 'training_chunks')
 
@@ -88,6 +89,7 @@ def selfplay(readouts=1600, verbose=2, resign_threshold=0.99):
     main.selfplay(
         load_file=model_save_file,
         output_dir=os.path.join(SELFPLAY_DIR, model_name),
+        holdout_dir=os.path.join(HOLDOUT_DIR, model_name),
         output_sgf=SGF_DIR,
         readouts=readouts,
         verbose=verbose,