update selfplay to write 5% of games to a holdout directory (#57)

* update selfplay to write 5% of games to a holdout directory

* comments

Fixes #43

main.py

@@ -118,10 +118,12 @@ def evaluate(
 def selfplay(
         load_file: "The path to the network model files",
         output_dir: "Where to write the games"="data/selfplay",
+        holdout_dir: "Where to write the games"="data/holdout",
         output_sgf: "Where to write the sgfs"="sgf/",
         readouts: 'How many simulations to run per move'=100,
         verbose: '>=2 will print debug info, >=3 will print boards' = 1,
-        resign_threshold: 'absolute value of threshold to resign at' = 0.95):
+        resign_threshold: 'absolute value of threshold to resign at' = 0.95
+        holdout_pct: 'how many games to hold out for evaluation' = 0.05):
     _ensure_dir_exists(output_sgf)
     _ensure_dir_exists(output_dir)
 
@@ -138,8 +140,15 @@ def selfplay(
     with gfile.GFile(os.path.join(output_sgf, '{}.sgf'.format(output_name)), 'w') as f:
         f.write(player.to_sgf())
 
-    fname = os.path.join(output_dir, "{}.tfrecord.zz".format(output_name))
-    preprocessing.make_dataset_from_selfplay(game_data, fname)
+    tf_examples = preprocessing.make_dataset_from_selfplay(game_data)
+
+    # Hold out 5% of games for evaluation.
+    if random.random() < holdout_pct:
+        fname = os.path.join(holdout_dir, "{}.tfrecord.zz".format(output_name))
+    else:
+        fname = os.path.join(output_dir, "{}.tfrecord.zz".format(output_name))
+
+    preprocessing.write_tf_examples(fname, tf_examples)
 
 
 def gather(

preprocessing.py

@@ -161,15 +161,15 @@ def get_input_tensors(batch_size, tf_records, num_repeats=None,
 # End-to-end utility functions
 
 
-def make_dataset_from_selfplay(data_extracts, tf_record):
+def make_dataset_from_selfplay(data_extracts):
     '''
+    Returns an iterable of tf.Examples.
     Args:
         data_extracts: An iterable of (position, pi, result) tuples
-        tf_record: name of file to write to
     '''
     tf_examples = (make_tf_example(features_lib.extract_features(pos), pi, result)
                    for pos, pi, result in data_extracts)
-    write_tf_examples(tf_record, tf_examples)
+    return tf_examples
 
 
 def make_dataset_from_sgf(sgf_filename, tf_record):

rl_loop.py

@@ -27,7 +27,8 @@
 
 BASE_DIR = "gs://{}".format(BUCKET_NAME)
 MODELS_DIR = os.path.join(BASE_DIR, 'models')
-SELFPLAY_DIR = os.path.join(BASE_DIR, 'games')
+SELFPLAY_DIR = os.path.join(BASE_DIR, 'data/selfplay')
+HOLDOUT_DIR = os.path.join(BASE_DIR, 'data/holdout')
 SGF_DIR = os.path.join(BASE_DIR, 'sgf')
 TRAINING_CHUNK_DIR = os.path.join(BASE_DIR, 'data', 'training_chunks')
 
@@ -88,6 +89,7 @@ def selfplay(readouts=1600, verbose=2, resign_threshold=0.99):
     main.selfplay(
         load_file=model_save_file,
         output_dir=os.path.join(SELFPLAY_DIR, model_name),
+        holdout_dir=os.path.join(HOLDOUT_DIR, model_name),
         output_sgf=SGF_DIR,
         readouts=readouts,
         verbose=verbose,