Update roaring + fix memory leaks (suggest-go#61)

* Update roaring package

* Fix memory leak + add error wrappers

go.mod

@@ -3,7 +3,7 @@ module github.com/suggest-go/suggest
 go 1.13
 
 require (
-	github.com/RoaringBitmap/roaring v0.5.0
+	github.com/RoaringBitmap/roaring v0.5.5
 	github.com/alldroll/cdb v1.0.2
 	github.com/alldroll/rbtree v0.0.0-20201026153457-c76906afcaa0
 	github.com/edsrzf/mmap-go v0.0.0-20190108065903-904c4ced31cd
@@ -14,6 +14,5 @@ require (
 	github.com/spf13/cobra v0.0.3
 	github.com/spf13/pflag v1.0.3 // indirect
 	github.com/stretchr/testify v1.6.1
-	golang.org/x/sync v0.0.0-20190423024810-112230192c58
-	golang.org/x/sys v0.0.0-20190426135247-a129542de9ae // indirect
+	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
 )

go.sum

@@ -1,5 +1,7 @@
 github.com/RoaringBitmap/roaring v0.5.0 h1:0psZZWU0J2AUl29BAylpHAsuBEEhCEfTKl2v5yHtXIg=
 github.com/RoaringBitmap/roaring v0.5.0/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo=
+github.com/RoaringBitmap/roaring v0.5.5 h1:naNqvO1mNnghk2UvcsqnzHDBn9DRbCIRy94GmDTRVTQ=
+github.com/RoaringBitmap/roaring v0.5.5/go.mod h1:puNo5VdzwbaIQxSiDIwfXl4Hnc+fbovcX4IW/dSTtUk=
 github.com/alldroll/cdb v1.0.2 h1:pSB3BphsF0m2DqOZm+IFyNm38nz1R8kCg3DPCusPLQE=
 github.com/alldroll/cdb v1.0.2/go.mod h1:PK3VAN9pconusJqa4kzOupYg9QxOnmgU8AcBWhuZZdo=
 github.com/alldroll/rbtree v0.0.0-20201026153457-c76906afcaa0 h1:IRs8Y64CCc/GWRo0a4+NiWyFjF6TfRO5iZKTXyCM5B0=
@@ -46,11 +48,34 @@ github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU=
 github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
 github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc=
 github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190426135247-a129542de9ae h1:mQLHiymj/JXKnnjc62tb7nD5pZLs940/sXJu+Xp3DBA=
 golang.org/x/sys v0.0.0-20190426135247-a129542de9ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200928182047-19e03678916f/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=

pkg/merger/cp_merge.go

@@ -1,6 +1,7 @@
 package merger
 
 import (
+	"fmt"
 	"sort"
 	"sync"
 )
@@ -25,6 +26,9 @@ func (cp *cpMerge) Merge(rid Rid, threshold int, collector Collector) error {
 	tmp := bufPool.Get().([]MergeCandidate)
 	candidates := bufPool.Get().([]MergeCandidate)
 
+	defer bufPool.Put(tmp[:0])
+	defer bufPool.Put(candidates[:0])
+
 	for _, list := range rid[:minQueries] {
 		isValid := true
 		current, err := list.Get()
@@ -33,7 +37,7 @@ func (cp *cpMerge) Merge(rid Rid, threshold int, collector Collector) error {
 			if err == ErrIteratorIsNotDereferencable {
 				isValid = false
 			} else {
-				return err
+				return fmt.Errorf("failed to call list.Get: %w", err)
 			}
 		}
 
@@ -48,7 +52,7 @@ func (cp *cpMerge) Merge(rid Rid, threshold int, collector Collector) error {
 					current, err = list.Next()
 
 					if err != nil {
-						return err
+						return fmt.Errorf("failed to call list.Next: %w", err)
 					}
 				} else {
 					isValid = false
@@ -65,7 +69,7 @@ func (cp *cpMerge) Merge(rid Rid, threshold int, collector Collector) error {
 					current, err = list.Next()
 
 					if err != nil {
-						return err
+						return fmt.Errorf("failed to call list.Next: %w", err)
 					}
 				} else {
 					isValid = false
@@ -76,7 +80,7 @@ func (cp *cpMerge) Merge(rid Rid, threshold int, collector Collector) error {
 		candidates, tmp = tmp, candidates
 	}
 
-	for i := minQueries; i < lenRid; i++ {
+	for i := minQueries; i < lenRid && len(candidates) > 0; i++ {
 		tmp = tmp[:0]
 
 		for _, c := range candidates {
@@ -87,7 +91,7 @@ func (cp *cpMerge) Merge(rid Rid, threshold int, collector Collector) error {
 			}
 
 			if err != nil && err != ErrIteratorIsNotDereferencable {
-				return err
+				return fmt.Errorf("failed to call list.LowerBound: %w", err)
 			}
 
 			if c.Overlap()+(lenRid-i-1) >= threshold {
@@ -96,10 +100,6 @@ func (cp *cpMerge) Merge(rid Rid, threshold int, collector Collector) error {
 		}
 
 		candidates, tmp = tmp, candidates
-
-		if len(candidates) == 0 {
-			break
-		}
 	}
 
 	for _, c := range candidates {
@@ -111,14 +111,11 @@ func (cp *cpMerge) Merge(rid Rid, threshold int, collector Collector) error {
 			}
 
 			if err != nil {
-				return err
+				return fmt.Errorf("failed to call collector.Collect: %w", err)
 			}
 		}
 	}
 
-	bufPool.Put(tmp[:0])
-	bufPool.Put(candidates[:0])
-
 	return nil
 }