boltdbshipper: fix crash of "timeout" panic when boltdbshipper do init

pkg/storage/stores/shipper/util/util.go

@@ -7,14 +7,14 @@ import (
 	"os"
 	"runtime/debug"
 	"strings"
+	"time"
 
-	"github.com/cortexproject/cortex/pkg/chunk/local"
+	util_log "github.com/cortexproject/cortex/pkg/util/log"
+	"github.com/go-kit/kit/log/level"
+	"github.com/pkg/errors"
 	"go.etcd.io/bbolt"
 
 	"github.com/grafana/loki/pkg/chunkenc"
-
-	util_log "github.com/cortexproject/cortex/pkg/util/log"
-	"github.com/go-kit/kit/log/level"
 )
 
 type StorageClient interface {
@@ -142,16 +142,37 @@ func safeOpenBoltDbFile(path string, ret chan *result) {
 	debug.SetPanicOnFault(true)
 	res := &result{}
 
+	var file *os.File
 	defer func() {
 		if r := recover(); r != nil {
 			res.err = fmt.Errorf("recovered from panic opening boltdb file: %v", r)
+
+			// A panic when opening the boltdb file may have left a lingering flock
+			// at the OS level, attempt to clear the flock that may exist
+			attemptCleanFlock(file)
 		}
 
 		// Return the result object on the channel to unblock the calling thread
 		ret <- res
 	}()
 
-	b, err := local.OpenBoltdbFile(path)
+	b, err := openBoltdbFile(path, func(s string, i int, mode os.FileMode) (f *os.File, err error) {
+		file, err = os.OpenFile(s, i, mode)
+		return file, err
+	})
 	res.boltdb = b
 	res.err = err
 }
+
+func openBoltdbFile(path string, f func(string, int, os.FileMode) (*os.File, error)) (*bbolt.DB, error) {
+	return bbolt.Open(path, 0666, &bbolt.Options{Timeout: 5 * time.Second, OpenFile: f})
+}
+
+// open a dummy db with previous opened file
+// intentionally throw an error using the provided OpenFile function
+// the error will trigger the flock cleanup within boltdb and should free all resources
+func attemptCleanFlock(f *os.File) {
+	_, _ = openBoltdbFile("", func(string, int, os.FileMode) (*os.File, error) {
+		return f, errors.New("error for cleanup")
+	})
+}