Fix bug: Do not add empty values to the term dictionary

FinalRep · Dec 3, 2019 · 571a36c · 571a36c
1 parent 057a136
commit 571a36c
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 4 deletions.
diff --git a/pkg/lm/binary.go b/pkg/lm/binary.go
@@ -162,10 +162,16 @@ func (dr *dictionaryReader) Iterate(iterator dictionary.Iterator) error {
 			return err
 		}
 
-		_, _ = tree.Insert(&dictItem{
+		if len(line[:tabIndex]) == 0 {
+			continue
+		}
+
+		item := &dictItem{
 			word:  line[:tabIndex],
 			count: WordCount(count),
-		})
+		}
+
+		_, _ = tree.Insert(item)
 	}
 
 	if err := lineScanner.Err(); err != nil {

diff --git a/pkg/lm/ngram_reader.go b/pkg/lm/ngram_reader.go
@@ -66,6 +66,7 @@ func (gr *googleNGramFormatReader) readNGramVector(builder NGramVectorBuilder, o
 	for scanner.Scan() {
 		line := scanner.Text()
 		tabIndex := strings.Index(line, "\t")
+		nGrams = nGrams[:0]
 
 		for _, word := range strings.Split(line[:tabIndex], " ") {
 			index, err := gr.indexer.Get(word)
@@ -86,8 +87,6 @@ func (gr *googleNGramFormatReader) readNGramVector(builder NGramVectorBuilder, o
 		if err := builder.Put(nGrams, WordCount(count)); err != nil {
 			return fmt.Errorf("failed to add nGrams to a builder: %v", err)
 		}
-
-		nGrams = nGrams[:0]
 	}
 
 	if err := scanner.Err(); err != nil {