skip sample if we hit a sample which has no content

stanfordnlp · Oct 23, 2023 · d0a8934 · d0a8934
1 parent 1223a18
commit d0a8934
Showing 1 changed file with 8 additions and 0 deletions.
diff --git a/stanza/models/pos/data.py b/stanza/models/pos/data.py
@@ -1,3 +1,4 @@
+from os import set_inheritable
 import random
 import logging
 import torch
@@ -196,6 +197,13 @@ def __getitem__(self, key):
         # get each character from the input sentnece
         chars = [w for sent in char for w in sent]
 
+        # augmentation, etc. resulted in nothing; we return a different sample
+        if len(chars) == 0:
+            if key == 0:
+                return self[-1]
+            else: 
+                return self[key-1]
+
         return DataSample(words, chars, upos, xpos, ufeats, pretrained, sample[6]), key
 
     def __iter__(self):