Chunk dev data into batches of args['batch_size']

stanfordnlp · Oct 22, 2023 · f9897ae · f9897ae
1 parent f450f3b
commit f9897ae
Showing 1 changed file with 1 addition and 2 deletions.
diff --git a/stanza/models/tagger.py b/stanza/models/tagger.py
@@ -231,8 +231,7 @@ def train(args):
                      for i in train_data]
     dev_doc = CoNLL.conll2doc(input_file=args['eval_file'])
     dev_data = Dataset(dev_doc, args, pretrain, vocab=vocab, evaluation=True, sort_during_eval=True)
-    # we want to put the entirety of the dev data into one batch
-    dev_batch = [next(iter(dev_data.to_loader(batch_size=len(dev_data), shuffle=True)))]
+    dev_batch = dev_data.to_loader(batch_size=args["batch_size"])
 
     eval_type = get_eval_type(dev_data)