From 192accd7aea4b66a73fe7df02e19faeb35bd1101 Mon Sep 17 00:00:00 2001 From: cool-developer <51834436+cool-develope@users.noreply.github.com> Date: Thu, 25 Jan 2024 12:49:44 -0500 Subject: [PATCH 1/3] feat: decouple cosmos-db (#874) (cherry picked from commit 11ba4961dae90838c95772f711fdbbd77de94098) # Conflicts: # .github/workflows/lint.yml # batch.go # nodedb.go --- .github/workflows/lint.yml | 8 + Makefile | 2 +- basic_test.go | 9 +- batch.go | 4 + batch_test.go | 9 +- benchmarks/bench_test.go | 16 +- benchmarks/cosmos-exim/main.go | 8 +- cmd/iaviewer/main.go | 3 +- db/memdb.go | 462 +++++++++++++++++++++++++++++++++ db/types.go | 137 ++++++++++ db/wrapper.go | 44 ++++ diff_test.go | 5 +- export.go | 4 +- export_test.go | 14 +- go.mod | 2 +- immutable_tree.go | 3 +- import_test.go | 30 +-- internal/bytes/bytes.go | 30 +++ iterator_test.go | 3 +- migrate_test.go | 9 +- mock/db_mock.go | 128 ++------- mutable_tree.go | 2 +- mutable_tree_test.go | 45 ++-- nodedb.go | 42 +-- nodedb_test.go | 26 +- proof_iavl_test.go | 5 +- proof_ics23_test.go | 4 +- testutils_test.go | 11 +- tree_random_test.go | 6 +- tree_test.go | 38 +-- 30 files changed, 862 insertions(+), 247 deletions(-) create mode 100644 db/memdb.go create mode 100644 db/types.go create mode 100644 db/wrapper.go diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 1e03feecc..b298a8d00 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -17,6 +17,14 @@ jobs: - name: 🐿 Setup Golang uses: actions/setup-go@v4 with: +<<<<<<< HEAD go-version: 1.21 - name: golangci-lint run: make lint +======= + go-version: "^1.20.0" + - name: golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + version: v1.55.2 +>>>>>>> 11ba496 (feat: decouple cosmos-db (#874)) diff --git a/Makefile b/Makefile index b23af8b26..73e05df27 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ format: # look into .golangci.yml for enabling / disabling linters golangci_lint_cmd=golangci-lint -golangci_version=v1.51.2 +golangci_version=v1.55.2 lint: @echo "--> Running linter" diff --git a/basic_test.go b/basic_test.go index b344a3212..e1e73cdd7 100644 --- a/basic_test.go +++ b/basic_test.go @@ -8,10 +8,11 @@ import ( "testing" "cosmossdk.io/log" - db "github.com/cosmos/cosmos-db" - iavlrand "github.com/cosmos/iavl/internal/rand" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + dbm "github.com/cosmos/iavl/db" + iavlrand "github.com/cosmos/iavl/internal/rand" ) func TestBasic(t *testing.T) { @@ -432,7 +433,7 @@ func TestIterateRange(t *testing.T) { } func TestPersistence(t *testing.T) { - db := db.NewMemDB() + db := dbm.NewMemDB() // Create some random key value pairs records := make(map[string]string) @@ -495,7 +496,7 @@ func TestProof(t *testing.T) { } func TestTreeProof(t *testing.T) { - db := db.NewMemDB() + db := dbm.NewMemDB() tree := NewMutableTree(db, 100, false, log.NewNopLogger()) hash := tree.Hash() assert.Equal(t, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", hex.EncodeToString(hash)) diff --git a/batch.go b/batch.go index a7e5e20b0..50f2a91aa 100644 --- a/batch.go +++ b/batch.go @@ -1,9 +1,13 @@ package iavl import ( +<<<<<<< HEAD "sync" dbm "github.com/cosmos/cosmos-db" +======= + dbm "github.com/cosmos/iavl/db" +>>>>>>> 11ba496 (feat: decouple cosmos-db (#874)) ) // BatchWithFlusher is a wrapper diff --git a/batch_test.go b/batch_test.go index 54ed0f2a2..d89d919e5 100644 --- a/batch_test.go +++ b/batch_test.go @@ -7,8 +7,9 @@ import ( "path/filepath" "testing" - dbm "github.com/cosmos/cosmos-db" "github.com/stretchr/testify/require" + + dbm "github.com/cosmos/iavl/db" ) func cleanupDBDir(dir, name string) { @@ -27,8 +28,8 @@ func makeKey(n uint16) []byte { } func TestBatchWithFlusher(t *testing.T) { - testedBackends := []dbm.BackendType{ - dbm.GoLevelDBBackend, + testedBackends := []string{ + "goleveldb", } for _, backend := range testedBackends { @@ -36,7 +37,7 @@ func TestBatchWithFlusher(t *testing.T) { } } -func testBatchWithFlusher(t *testing.T, backend dbm.BackendType) { +func testBatchWithFlusher(t *testing.T, backend string) { name := fmt.Sprintf("test_%x", randstr(12)) dir := t.TempDir() db, err := dbm.NewDB(name, backend, dir) diff --git a/benchmarks/bench_test.go b/benchmarks/bench_test.go index 595da1c4f..8a82590d6 100644 --- a/benchmarks/bench_test.go +++ b/benchmarks/bench_test.go @@ -12,8 +12,8 @@ import ( "cosmossdk.io/log" "github.com/stretchr/testify/require" - db "github.com/cosmos/cosmos-db" "github.com/cosmos/iavl" + dbm "github.com/cosmos/iavl/db" ) const historySize = 20 @@ -26,7 +26,7 @@ func randBytes(length int) []byte { return key } -func prepareTree(b *testing.B, db db.DB, size, keyLen, dataLen int) (*iavl.MutableTree, [][]byte) { +func prepareTree(b *testing.B, db dbm.DB, size, keyLen, dataLen int) (*iavl.MutableTree, [][]byte) { t := iavl.NewMutableTree(db, size, false, log.NewNopLogger()) keys := make([][]byte, size) @@ -147,7 +147,7 @@ func runIterationSlow(b *testing.B, t *iavl.MutableTree, expectedSize int) { } } -func iterate(b *testing.B, itr db.Iterator, expectedSize int) { +func iterate(b *testing.B, itr dbm.Iterator, expectedSize int) { b.StartTimer() keyValuePairs := make([][][]byte, 0, expectedSize) for i := 0; i < expectedSize && itr.Valid(); i++ { @@ -259,7 +259,7 @@ func BenchmarkRandomBytes(b *testing.B) { } type benchmark struct { - dbType db.BackendType + dbType string initSize, blockSize int keyLen, dataLen int } @@ -330,7 +330,7 @@ func runBenchmarks(b *testing.B, benchmarks []benchmark) { // prepare a dir for the db and cleanup afterwards dirName := fmt.Sprintf("./%s-db", prefix) - if bb.dbType == db.RocksDBBackend { + if bb.dbType == "rocksdb" { _ = os.Mkdir(dirName, 0o755) } @@ -343,11 +343,11 @@ func runBenchmarks(b *testing.B, benchmarks []benchmark) { // note that "" leads to nil backing db! var ( - d db.DB + d dbm.DB err error ) if bb.dbType != "nodb" { - d, err = db.NewDB("test", bb.dbType, dirName) + d, err = dbm.NewDB("test", bb.dbType, dirName) if err != nil { if strings.Contains(err.Error(), "unknown db_backend") { @@ -376,7 +376,7 @@ func memUseMB() float64 { return mb } -func runSuite(b *testing.B, d db.DB, initSize, blockSize, keyLen, dataLen int) { +func runSuite(b *testing.B, d dbm.DB, initSize, blockSize, keyLen, dataLen int) { // measure mem usage runtime.GC() init := memUseMB() diff --git a/benchmarks/cosmos-exim/main.go b/benchmarks/cosmos-exim/main.go index 98d629923..408ad08ca 100644 --- a/benchmarks/cosmos-exim/main.go +++ b/benchmarks/cosmos-exim/main.go @@ -7,7 +7,9 @@ import ( "cosmossdk.io/log" tmdb "github.com/cosmos/cosmos-db" + "github.com/cosmos/iavl" + idbm "github.com/cosmos/iavl/db" ) // stores is the list of stores in the CosmosHub database @@ -91,7 +93,7 @@ func runExport(dbPath string) (int64, map[string][]*iavl.ExportNode, error) { if err != nil { return 0, nil, err } - tree := iavl.NewMutableTree(tmdb.NewPrefixDB(ldb, []byte("s/k:main/")), 0, false, log.NewNopLogger()) + tree := iavl.NewMutableTree(idbm.NewWrapper(tmdb.NewPrefixDB(ldb, []byte("s/k:main/"))), 0, false, log.NewNopLogger()) version, err := tree.LoadVersion(0) if err != nil { return 0, nil, err @@ -103,7 +105,7 @@ func runExport(dbPath string) (int64, map[string][]*iavl.ExportNode, error) { totalStats := Stats{} for _, name := range stores { db := tmdb.NewPrefixDB(ldb, []byte("s/k:"+name+"/")) - tree := iavl.NewMutableTree(db, 0, false, log.NewNopLogger()) + tree := iavl.NewMutableTree(idbm.NewWrapper(db), 0, false, log.NewNopLogger()) stats := Stats{} export := make([]*iavl.ExportNode, 0, 100000) @@ -168,7 +170,7 @@ func runImport(version int64, exports map[string][]*iavl.ExportNode) error { if err != nil { return err } - newTree := iavl.NewMutableTree(newDB, 0, false, log.NewNopLogger()) + newTree := iavl.NewMutableTree(idbm.NewWrapper(newDB), 0, false, log.NewNopLogger()) importer, err := newTree.Import(version) if err != nil { return err diff --git a/cmd/iaviewer/main.go b/cmd/iaviewer/main.go index 6a8ba2982..796741698 100644 --- a/cmd/iaviewer/main.go +++ b/cmd/iaviewer/main.go @@ -14,6 +14,7 @@ import ( dbm "github.com/cosmos/cosmos-db" "github.com/cosmos/iavl" + idbm "github.com/cosmos/iavl/db" ) // TODO: make this configurable? @@ -122,7 +123,7 @@ func ReadTree(dir string, version int, prefix []byte) (*iavl.MutableTree, error) db = dbm.NewPrefixDB(db, prefix) } - tree := iavl.NewMutableTree(db, DefaultCacheSize, false, log.NewLogger(os.Stdout)) + tree := iavl.NewMutableTree(idbm.NewWrapper(db), DefaultCacheSize, false, log.NewLogger(os.Stdout)) ver, err := tree.LoadVersion(int64(version)) fmt.Printf("Got version: %d\n", ver) return tree, err diff --git a/db/memdb.go b/db/memdb.go new file mode 100644 index 000000000..614c52f94 --- /dev/null +++ b/db/memdb.go @@ -0,0 +1,462 @@ +package db + +import ( + "bytes" + "context" + "fmt" + "sync" + + "github.com/google/btree" +) + +const ( + // The approximate number of items and children per B-tree node. Tuned with benchmarks. + bTreeDegree = 32 +) + +// item is a btree.Item with byte slices as keys and values +type item struct { + key []byte + value []byte +} + +// Less implements btree.Item. +func (i item) Less(other btree.Item) bool { + // this considers nil == []byte{}, but that's ok since we handle nil endpoints + // in iterators specially anyway + return bytes.Compare(i.key, other.(item).key) == -1 +} + +// newKey creates a new key item. +func newKey(key []byte) item { + return item{key: key} +} + +// newPair creates a new pair item. +func newPair(key, value []byte) item { + return item{key: key, value: value} +} + +// MemDB is an in-memory database backend using a B-tree for the test purpose. +// +// For performance reasons, all given and returned keys and values are pointers to the in-memory +// database, so modifying them will cause the stored values to be modified as well. All DB methods +// already specify that keys and values should be considered read-only, but this is especially +// important with MemDB. +type MemDB struct { + mtx sync.RWMutex + btree *btree.BTree +} + +var _ DB = (*MemDB)(nil) + +// NewMemDB creates a new in-memory database. +func NewMemDB() *MemDB { + database := &MemDB{ + btree: btree.New(bTreeDegree), + } + return database +} + +// Get implements DB. +func (db *MemDB) Get(key []byte) ([]byte, error) { + if len(key) == 0 { + return nil, errKeyEmpty + } + db.mtx.RLock() + defer db.mtx.RUnlock() + + i := db.btree.Get(newKey(key)) + if i != nil { + return i.(item).value, nil + } + return nil, nil +} + +// Has implements DB. +func (db *MemDB) Has(key []byte) (bool, error) { + if len(key) == 0 { + return false, errKeyEmpty + } + db.mtx.RLock() + defer db.mtx.RUnlock() + + return db.btree.Has(newKey(key)), nil +} + +// Set implements DB. +func (db *MemDB) Set(key []byte, value []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if value == nil { + return errValueNil + } + db.mtx.Lock() + defer db.mtx.Unlock() + + db.set(key, value) + return nil +} + +// set sets a value without locking the mutex. +func (db *MemDB) set(key []byte, value []byte) { + db.btree.ReplaceOrInsert(newPair(key, value)) +} + +// SetSync implements DB. +func (db *MemDB) SetSync(key []byte, value []byte) error { + return db.Set(key, value) +} + +// Delete implements DB. +func (db *MemDB) Delete(key []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + db.mtx.Lock() + defer db.mtx.Unlock() + + db.delete(key) + return nil +} + +// delete deletes a key without locking the mutex. +func (db *MemDB) delete(key []byte) { + db.btree.Delete(newKey(key)) +} + +// DeleteSync implements DB. +func (db *MemDB) DeleteSync(key []byte) error { + return db.Delete(key) +} + +// Close implements DB. +func (db *MemDB) Close() error { + // Close is a noop since for an in-memory database, we don't have a destination to flush + // contents to nor do we want any data loss on invoking Close(). + // See the discussion in https://github.com/tendermint/tendermint/libs/pull/56 + return nil +} + +// Print implements DB. +func (db *MemDB) Print() error { + db.mtx.RLock() + defer db.mtx.RUnlock() + + db.btree.Ascend(func(i btree.Item) bool { + item := i.(item) + fmt.Printf("[%X]:\t[%X]\n", item.key, item.value) + return true + }) + return nil +} + +// Stats implements DB. +func (db *MemDB) Stats() map[string]string { + db.mtx.RLock() + defer db.mtx.RUnlock() + + stats := make(map[string]string) + stats["database.type"] = "memDB" + stats["database.size"] = fmt.Sprintf("%d", db.btree.Len()) + return stats +} + +// NewBatch implements DB. +func (db *MemDB) NewBatch() Batch { + return newMemDBBatch(db) +} + +// NewBatchWithSize implements DB. +// It does the same thing as NewBatch because we can't pre-allocate memDBBatch +func (db *MemDB) NewBatchWithSize(_ int) Batch { + return newMemDBBatch(db) +} + +// Iterator implements DB. +// Takes out a read-lock on the database until the iterator is closed. +func (db *MemDB) Iterator(start, end []byte) (Iterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, errKeyEmpty + } + return newMemDBIterator(db, start, end, false), nil +} + +// ReverseIterator implements DB. +// Takes out a read-lock on the database until the iterator is closed. +func (db *MemDB) ReverseIterator(start, end []byte) (Iterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, errKeyEmpty + } + return newMemDBIterator(db, start, end, true), nil +} + +// IteratorNoMtx makes an iterator with no mutex. +func (db *MemDB) IteratorNoMtx(start, end []byte) (Iterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, errKeyEmpty + } + return newMemDBIteratorMtxChoice(db, start, end, false, false), nil +} + +// ReverseIteratorNoMtx makes an iterator with no mutex. +func (db *MemDB) ReverseIteratorNoMtx(start, end []byte) (Iterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, errKeyEmpty + } + return newMemDBIteratorMtxChoice(db, start, end, true, false), nil +} + +const ( + // Size of the channel buffer between traversal goroutine and iterator. Using an unbuffered + // channel causes two context switches per item sent, while buffering allows more work per + // context switch. Tuned with benchmarks. + chBufferSize = 64 +) + +// memDBIterator is a memDB iterator. +type memDBIterator struct { + ch <-chan *item + cancel context.CancelFunc + item *item + start []byte + end []byte + useMtx bool +} + +var _ Iterator = (*memDBIterator)(nil) + +// newMemDBIterator creates a new memDBIterator. +func newMemDBIterator(db *MemDB, start []byte, end []byte, reverse bool) *memDBIterator { + return newMemDBIteratorMtxChoice(db, start, end, reverse, true) +} + +func newMemDBIteratorMtxChoice(db *MemDB, start []byte, end []byte, reverse bool, useMtx bool) *memDBIterator { + ctx, cancel := context.WithCancel(context.Background()) + ch := make(chan *item, chBufferSize) + iter := &memDBIterator{ + ch: ch, + cancel: cancel, + start: start, + end: end, + useMtx: useMtx, + } + + if useMtx { + db.mtx.RLock() + } + go func() { + if useMtx { + defer db.mtx.RUnlock() + } + // Because we use [start, end) for reverse ranges, while btree uses (start, end], we need + // the following variables to handle some reverse iteration conditions ourselves. + var ( + skipEqual []byte + abortLessThan []byte + ) + visitor := func(i btree.Item) bool { + item := i.(item) + if skipEqual != nil && bytes.Equal(item.key, skipEqual) { + skipEqual = nil + return true + } + if abortLessThan != nil && bytes.Compare(item.key, abortLessThan) == -1 { + return false + } + select { + case <-ctx.Done(): + return false + case ch <- &item: + return true + } + } + switch { + case start == nil && end == nil && !reverse: + db.btree.Ascend(visitor) + case start == nil && end == nil && reverse: + db.btree.Descend(visitor) + case end == nil && !reverse: + // must handle this specially, since nil is considered less than anything else + db.btree.AscendGreaterOrEqual(newKey(start), visitor) + case !reverse: + db.btree.AscendRange(newKey(start), newKey(end), visitor) + case end == nil: + // abort after start, since we use [start, end) while btree uses (start, end] + abortLessThan = start + db.btree.Descend(visitor) + default: + // skip end and abort after start, since we use [start, end) while btree uses (start, end] + skipEqual = end + abortLessThan = start + db.btree.DescendLessOrEqual(newKey(end), visitor) + } + close(ch) + }() + + // prime the iterator with the first value, if any + if item, ok := <-ch; ok { + iter.item = item + } + + return iter +} + +// Close implements Iterator. +func (i *memDBIterator) Close() error { + i.cancel() + for range i.ch { //nolint:revive + } // drain channel + i.item = nil + return nil +} + +// Domain implements Iterator. +func (i *memDBIterator) Domain() ([]byte, []byte) { + return i.start, i.end +} + +// Valid implements Iterator. +func (i *memDBIterator) Valid() bool { + return i.item != nil +} + +// Next implements Iterator. +func (i *memDBIterator) Next() { + i.assertIsValid() + item, ok := <-i.ch + switch { + case ok: + i.item = item + default: + i.item = nil + } +} + +// Error implements Iterator. +func (i *memDBIterator) Error() error { + return nil // famous last words +} + +// Key implements Iterator. +func (i *memDBIterator) Key() []byte { + i.assertIsValid() + return i.item.key +} + +// Value implements Iterator. +func (i *memDBIterator) Value() []byte { + i.assertIsValid() + return i.item.value +} + +func (i *memDBIterator) assertIsValid() { + if !i.Valid() { + panic("iterator is invalid") + } +} + +// memDBBatch operations +type opType int + +const ( + opTypeSet opType = iota + 1 + opTypeDelete +) + +type operation struct { + opType + key []byte + value []byte +} + +// memDBBatch handles in-memory batching. +type memDBBatch struct { + db *MemDB + ops []operation + size int +} + +var _ Batch = (*memDBBatch)(nil) + +// newMemDBBatch creates a new memDBBatch +func newMemDBBatch(db *MemDB) *memDBBatch { + return &memDBBatch{ + db: db, + ops: []operation{}, + size: 0, + } +} + +// Set implements Batch. +func (b *memDBBatch) Set(key, value []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if value == nil { + return errValueNil + } + if b.ops == nil { + return errBatchClosed + } + b.size += len(key) + len(value) + b.ops = append(b.ops, operation{opTypeSet, key, value}) + return nil +} + +// Delete implements Batch. +func (b *memDBBatch) Delete(key []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if b.ops == nil { + return errBatchClosed + } + b.size += len(key) + b.ops = append(b.ops, operation{opTypeDelete, key, nil}) + return nil +} + +// Write implements Batch. +func (b *memDBBatch) Write() error { + if b.ops == nil { + return errBatchClosed + } + b.db.mtx.Lock() + defer b.db.mtx.Unlock() + + for _, op := range b.ops { + switch op.opType { + case opTypeSet: + b.db.set(op.key, op.value) + case opTypeDelete: + b.db.delete(op.key) + default: + return fmt.Errorf("unknown operation type %v (%v)", op.opType, op) + } + } + + // Make sure batch cannot be used afterwards. Callers should still call Close(), for errors. + return b.Close() +} + +// WriteSync implements Batch. +func (b *memDBBatch) WriteSync() error { + return b.Write() +} + +// Close implements Batch. +func (b *memDBBatch) Close() error { + b.ops = nil + b.size = 0 + return nil +} + +// GetByteSize implements Batch +func (b *memDBBatch) GetByteSize() (int, error) { + if b.ops == nil { + return 0, errBatchClosed + } + return b.size, nil +} diff --git a/db/types.go b/db/types.go new file mode 100644 index 000000000..840494f41 --- /dev/null +++ b/db/types.go @@ -0,0 +1,137 @@ +package db + +import "errors" + +var ( + // errBatchClosed is returned when a closed or written batch is used. + errBatchClosed = errors.New("batch has been written or closed") + + // errKeyEmpty is returned when attempting to use an empty or nil key. + errKeyEmpty = errors.New("key cannot be empty") + + // errValueNil is returned when attempting to set a nil value. + errValueNil = errors.New("value cannot be nil") +) + +// DB is the main interface for all database backends. DBs are concurrency-safe. Callers must call +// Close on the database when done. +// +// Keys cannot be nil or empty, while values cannot be nil. Keys and values should be considered +// read-only, both when returned and when given, and must be copied before they are modified. +type DB interface { + // Get fetches the value of the given key, or nil if it does not exist. + // CONTRACT: key, value readonly []byte + Get([]byte) ([]byte, error) + + // Has checks if a key exists. + // CONTRACT: key, value readonly []byte + Has(key []byte) (bool, error) + + // Iterator returns an iterator over a domain of keys, in ascending order. The caller must call + // Close when done. End is exclusive, and start must be less than end. A nil start iterates + // from the first key, and a nil end iterates to the last key (inclusive). Empty keys are not + // valid. + // CONTRACT: No writes may happen within a domain while an iterator exists over it. + // CONTRACT: start, end readonly []byte + Iterator(start, end []byte) (Iterator, error) + + // ReverseIterator returns an iterator over a domain of keys, in descending order. The caller + // must call Close when done. End is exclusive, and start must be less than end. A nil end + // iterates from the last key (inclusive), and a nil start iterates to the first key (inclusive). + // Empty keys are not valid. + // CONTRACT: No writes may happen within a domain while an iterator exists over it. + // CONTRACT: start, end readonly []byte + ReverseIterator(start, end []byte) (Iterator, error) + + // Close closes the database connection. + Close() error + + // NewBatch creates a batch for atomic updates. The caller must call Batch.Close. + NewBatch() Batch + + // NewBatchWithSize create a new batch for atomic updates, but with pre-allocated size. + // This will does the same thing as NewBatch if the batch implementation doesn't support pre-allocation. + NewBatchWithSize(int) Batch +} + +// Iterator represents an iterator over a domain of keys. Callers must call Close when done. +// No writes can happen to a domain while there exists an iterator over it, some backends may take +// out database locks to ensure this will not happen. +// +// Callers must make sure the iterator is valid before calling any methods on it, otherwise +// these methods will panic. This is in part caused by most backend databases using this convention. +// +// As with DB, keys and values should be considered read-only, and must be copied before they are +// modified. +// +// Typical usage: +// +// var itr Iterator = ... +// defer itr.Close() +// +// for ; itr.Valid(); itr.Next() { +// k, v := itr.Key(); itr.Value() +// ... +// } +// +// if err := itr.Error(); err != nil { +// ... +// } +type Iterator interface { + // Domain returns the start (inclusive) and end (exclusive) limits of the iterator. + // CONTRACT: start, end readonly []byte + Domain() (start []byte, end []byte) + + // Valid returns whether the current iterator is valid. Once invalid, the Iterator remains + // invalid forever. + Valid() bool + + // Next moves the iterator to the next key in the database, as defined by order of iteration. + // If Valid returns false, this method will panic. + Next() + + // Key returns the key at the current position. Panics if the iterator is invalid. + // CONTRACT: key readonly []byte + Key() (key []byte) + + // Value returns the value at the current position. Panics if the iterator is invalid. + // CONTRACT: value readonly []byte + Value() (value []byte) + + // Error returns the last error encountered by the iterator, if any. + Error() error + + // Close closes the iterator, relasing any allocated resources. + Close() error +} + +// Batch represents a group of writes. They may or may not be written atomically depending on the +// backend. Callers must call Close on the batch when done. +// +// As with DB, given keys and values should be considered read-only, and must not be modified after +// passing them to the batch. +type Batch interface { + // Set sets a key/value pair. + // CONTRACT: key, value readonly []byte + Set(key, value []byte) error + + // Delete deletes a key/value pair. + // CONTRACT: key readonly []byte + Delete(key []byte) error + + // Write writes the batch, possibly without flushing to disk. Only Close() can be called after, + // other methods will error. + Write() error + + // WriteSync writes the batch and flushes it to disk. Only Close() can be called after, other + // methods will error. + WriteSync() error + + // Close closes the batch. It is idempotent, but calls to other methods afterwards will error. + Close() error + + // GetByteSize that returns the current size of the batch in bytes. Depending on the implementation, + // this may return the size of the underlying LSM batch, including the size of additional metadata + // on top of the expected key and value total byte count. + GetByteSize() (int, error) +} diff --git a/db/wrapper.go b/db/wrapper.go new file mode 100644 index 000000000..fce80f692 --- /dev/null +++ b/db/wrapper.go @@ -0,0 +1,44 @@ +package db + +import dbm "github.com/cosmos/cosmos-db" + +// Wrapper wraps a dbm.DB to implement DB. +type Wrapper struct { + dbm.DB +} + +var _ DB = (*Wrapper)(nil) + +// NewWrapper returns a new Wrapper. +func NewWrapper(db dbm.DB) *Wrapper { + return &Wrapper{DB: db} +} + +// Iterator implements DB. +func (db *Wrapper) Iterator(start, end []byte) (Iterator, error) { + return db.DB.Iterator(start, end) +} + +// ReverseIterator implements DB. +func (db *Wrapper) ReverseIterator(start, end []byte) (Iterator, error) { + return db.DB.ReverseIterator(start, end) +} + +// NewBatch implements DB. +func (db *Wrapper) NewBatch() Batch { + return db.DB.NewBatch() +} + +// NewBatchWithSize implements DB. +func (db *Wrapper) NewBatchWithSize(size int) Batch { + return db.DB.NewBatchWithSize(size) +} + +// NewDB returns a new Wrapper. +func NewDB(name, backendType, dir string) (*Wrapper, error) { + db, err := dbm.NewDB(name, dbm.BackendType(backendType), dir) + if err != nil { + return nil, err + } + return NewWrapper(db), nil +} diff --git a/diff_test.go b/diff_test.go index 92f8f39e5..0cdf080f2 100644 --- a/diff_test.go +++ b/diff_test.go @@ -9,8 +9,9 @@ import ( "testing" "cosmossdk.io/log" - db "github.com/cosmos/cosmos-db" "github.com/stretchr/testify/require" + + dbm "github.com/cosmos/iavl/db" ) // TestDiffRoundTrip generate random change sets, build an iavl tree versions, @@ -19,7 +20,7 @@ func TestDiffRoundTrip(t *testing.T) { changeSets := genChangeSets(rand.New(rand.NewSource(0)), 300) // apply changeSets to tree - db := db.NewMemDB() + db := dbm.NewMemDB() tree := NewMutableTree(db, 0, true, log.NewNopLogger()) for i := range changeSets { v, err := tree.SaveChangeSet(changeSets[i]) diff --git a/export.go b/export.go index 81b5c10e9..af7f52649 100644 --- a/export.go +++ b/export.go @@ -90,8 +90,8 @@ func (e *Exporter) Next() (*ExportNode, error) { // Close closes the exporter. It is safe to call multiple times. func (e *Exporter) Close() { e.cancel() - for range e.ch { // drain channel - } + for range e.ch { //nolint:revive + } // drain channel if e.tree != nil { e.tree.ndb.decrVersionReaders(e.tree.version) } diff --git a/export_test.go b/export_test.go index af25aa799..3abd602e9 100644 --- a/export_test.go +++ b/export_test.go @@ -9,13 +9,13 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - db "github.com/cosmos/cosmos-db" + dbm "github.com/cosmos/iavl/db" ) // setupExportTreeBasic sets up a basic tree with a handful of // create/update/delete operations over a few versions. func setupExportTreeBasic(t require.TestingT) *ImmutableTree { - tree := NewMutableTree(db.NewMemDB(), 0, false, log.NewNopLogger()) + tree := NewMutableTree(dbm.NewMemDB(), 0, false, log.NewNopLogger()) _, err := tree.Set([]byte("x"), []byte{255}) require.NoError(t, err) @@ -74,7 +74,7 @@ func setupExportTreeRandom(t *testing.T) *ImmutableTree { ) r := rand.New(rand.NewSource(randSeed)) - tree := NewMutableTree(db.NewMemDB(), 0, false, log.NewNopLogger()) + tree := NewMutableTree(dbm.NewMemDB(), 0, false, log.NewNopLogger()) var version int64 keys := make([][]byte, 0, versionOps) @@ -134,7 +134,7 @@ func setupExportTreeSized(t require.TestingT, treeSize int) *ImmutableTree { ) r := rand.New(rand.NewSource(randSeed)) - tree := NewMutableTree(db.NewMemDB(), 0, false, log.NewNopLogger()) + tree := NewMutableTree(dbm.NewMemDB(), 0, false, log.NewNopLogger()) for i := 0; i < treeSize; i++ { key := make([]byte, keySize) @@ -228,7 +228,7 @@ func TestExporterCompress(t *testing.T) { func TestExporter_Import(t *testing.T) { testcases := map[string]*ImmutableTree{ - "empty tree": NewImmutableTree(db.NewMemDB(), 0, false, log.NewNopLogger()), + "empty tree": NewImmutableTree(dbm.NewMemDB(), 0, false, log.NewNopLogger()), "basic tree": setupExportTreeBasic(t), } if !testing.Short() { @@ -255,7 +255,7 @@ func TestExporter_Import(t *testing.T) { exporter = NewCompressExporter(innerExporter) } - newTree := NewMutableTree(db.NewMemDB(), 0, false, log.NewNopLogger()) + newTree := NewMutableTree(dbm.NewMemDB(), 0, false, log.NewNopLogger()) innerImporter, err := newTree.Import(tree.Version()) require.NoError(t, err) defer innerImporter.Close() @@ -323,7 +323,7 @@ func TestExporter_Close(t *testing.T) { } func TestExporter_DeleteVersionErrors(t *testing.T) { - tree := NewMutableTree(db.NewMemDB(), 0, false, log.NewNopLogger()) + tree := NewMutableTree(dbm.NewMemDB(), 0, false, log.NewNopLogger()) _, err := tree.Set([]byte("a"), []byte{1}) require.NoError(t, err) diff --git a/go.mod b/go.mod index 356a691a7..72d8e15df 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/cosmos/ics23/go v0.10.0 github.com/emicklei/dot v1.4.2 github.com/golang/mock v1.6.0 + github.com/google/btree v1.1.2 github.com/stretchr/testify v1.8.4 golang.org/x/crypto v0.12.0 google.golang.org/protobuf v1.30.0 @@ -27,7 +28,6 @@ require ( github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/btree v1.1.2 // indirect github.com/klauspost/compress v1.15.9 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect diff --git a/immutable_tree.go b/immutable_tree.go index dc8f4a508..7bff7077b 100644 --- a/immutable_tree.go +++ b/immutable_tree.go @@ -5,7 +5,8 @@ import ( "strings" "cosmossdk.io/log" - dbm "github.com/cosmos/cosmos-db" + + dbm "github.com/cosmos/iavl/db" ) // ImmutableTree contains the immutable tree at a given version. It is typically created by calling diff --git a/import_test.go b/import_test.go index c027b99cc..13a925be7 100644 --- a/import_test.go +++ b/import_test.go @@ -7,11 +7,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - db "github.com/cosmos/cosmos-db" + dbm "github.com/cosmos/iavl/db" ) func ExampleImporter() { - tree := NewMutableTree(db.NewMemDB(), 0, false, log.NewNopLogger()) + tree := NewMutableTree(dbm.NewMemDB(), 0, false, log.NewNopLogger()) _, err := tree.Set([]byte("a"), []byte{1}) if err != nil { @@ -52,7 +52,7 @@ func ExampleImporter() { exported = append(exported, node) } - newTree := NewMutableTree(db.NewMemDB(), 0, false, log.NewNopLogger()) + newTree := NewMutableTree(dbm.NewMemDB(), 0, false, log.NewNopLogger()) importer, err := newTree.Import(version) if err != nil { panic(err) @@ -71,13 +71,13 @@ func ExampleImporter() { } func TestImporter_NegativeVersion(t *testing.T) { - tree := NewMutableTree(db.NewMemDB(), 0, false, log.NewNopLogger()) + tree := NewMutableTree(dbm.NewMemDB(), 0, false, log.NewNopLogger()) _, err := tree.Import(-1) require.Error(t, err) } func TestImporter_NotEmpty(t *testing.T) { - tree := NewMutableTree(db.NewMemDB(), 0, false, log.NewNopLogger()) + tree := NewMutableTree(dbm.NewMemDB(), 0, false, log.NewNopLogger()) _, err := tree.Set([]byte("a"), []byte{1}) require.NoError(t, err) _, _, err = tree.SaveVersion() @@ -88,7 +88,7 @@ func TestImporter_NotEmpty(t *testing.T) { } func TestImporter_NotEmptyDatabase(t *testing.T) { - db := db.NewMemDB() + db := dbm.NewMemDB() tree := NewMutableTree(db, 0, false, log.NewNopLogger()) _, err := tree.Set([]byte("a"), []byte{1}) @@ -105,7 +105,7 @@ func TestImporter_NotEmptyDatabase(t *testing.T) { } func TestImporter_NotEmptyUnsaved(t *testing.T) { - tree := NewMutableTree(db.NewMemDB(), 0, false, log.NewNopLogger()) + tree := NewMutableTree(dbm.NewMemDB(), 0, false, log.NewNopLogger()) _, err := tree.Set([]byte("a"), []byte{1}) require.NoError(t, err) @@ -132,7 +132,7 @@ func TestImporter_Add(t *testing.T) { for desc, tc := range testcases { tc := tc // appease scopelint t.Run(desc, func(t *testing.T) { - tree := NewMutableTree(db.NewMemDB(), 0, false, log.NewNopLogger()) + tree := NewMutableTree(dbm.NewMemDB(), 0, false, log.NewNopLogger()) importer, err := tree.Import(1) require.NoError(t, err) defer importer.Close() @@ -151,7 +151,7 @@ func TestImporter_Add(t *testing.T) { } func TestImporter_Add_Closed(t *testing.T) { - tree := NewMutableTree(db.NewMemDB(), 0, false, log.NewNopLogger()) + tree := NewMutableTree(dbm.NewMemDB(), 0, false, log.NewNopLogger()) importer, err := tree.Import(1) require.NoError(t, err) @@ -162,7 +162,7 @@ func TestImporter_Add_Closed(t *testing.T) { } func TestImporter_Close(t *testing.T) { - tree := NewMutableTree(db.NewMemDB(), 0, false, log.NewNopLogger()) + tree := NewMutableTree(dbm.NewMemDB(), 0, false, log.NewNopLogger()) importer, err := tree.Import(1) require.NoError(t, err) @@ -178,7 +178,7 @@ func TestImporter_Close(t *testing.T) { } func TestImporter_Commit(t *testing.T) { - tree := NewMutableTree(db.NewMemDB(), 0, false, log.NewNopLogger()) + tree := NewMutableTree(dbm.NewMemDB(), 0, false, log.NewNopLogger()) importer, err := tree.Import(1) require.NoError(t, err) @@ -193,7 +193,7 @@ func TestImporter_Commit(t *testing.T) { } func TestImporter_Commit_ForwardVersion(t *testing.T) { - tree := NewMutableTree(db.NewMemDB(), 0, false, log.NewNopLogger()) + tree := NewMutableTree(dbm.NewMemDB(), 0, false, log.NewNopLogger()) importer, err := tree.Import(2) require.NoError(t, err) @@ -208,7 +208,7 @@ func TestImporter_Commit_ForwardVersion(t *testing.T) { } func TestImporter_Commit_Closed(t *testing.T) { - tree := NewMutableTree(db.NewMemDB(), 0, false, log.NewNopLogger()) + tree := NewMutableTree(dbm.NewMemDB(), 0, false, log.NewNopLogger()) importer, err := tree.Import(1) require.NoError(t, err) @@ -222,7 +222,7 @@ func TestImporter_Commit_Closed(t *testing.T) { } func TestImporter_Commit_Empty(t *testing.T) { - tree := NewMutableTree(db.NewMemDB(), 0, false, log.NewNopLogger()) + tree := NewMutableTree(dbm.NewMemDB(), 0, false, log.NewNopLogger()) importer, err := tree.Import(3) require.NoError(t, err) defer importer.Close() @@ -259,7 +259,7 @@ func benchmarkImport(b *testing.B, nodes int) { b.StartTimer() for n := 0; n < b.N; n++ { - newTree := NewMutableTree(db.NewMemDB(), 0, false, log.NewNopLogger()) + newTree := NewMutableTree(dbm.NewMemDB(), 0, false, log.NewNopLogger()) importer, err := newTree.Import(tree.Version()) require.NoError(b, err) for _, item := range exported { diff --git a/internal/bytes/bytes.go b/internal/bytes/bytes.go index ac3197d8a..23c042196 100644 --- a/internal/bytes/bytes.go +++ b/internal/bytes/bytes.go @@ -62,3 +62,33 @@ func (bz HexBytes) Format(s fmt.State, verb rune) { s.Write([]byte(fmt.Sprintf("%X", []byte(bz)))) } } + +// Returns a copy of the given byte slice. +func Cp(bz []byte) (ret []byte) { + ret = make([]byte, len(bz)) + copy(ret, bz) + return ret +} + +// Returns a slice of the same length (big endian) +// except incremented by one. +// Returns nil on overflow (e.g. if bz bytes are all 0xFF) +// CONTRACT: len(bz) > 0 +func CpIncr(bz []byte) (ret []byte) { + if len(bz) == 0 { + panic("cpIncr expects non-zero bz length") + } + ret = Cp(bz) + for i := len(bz) - 1; i >= 0; i-- { + if ret[i] < byte(0xFF) { + ret[i]++ + return + } + ret[i] = byte(0x00) + if i == 0 { + // Overflow + return nil + } + } + return nil +} diff --git a/iterator_test.go b/iterator_test.go index 6107dfa5f..d684bb8af 100644 --- a/iterator_test.go +++ b/iterator_test.go @@ -7,8 +7,9 @@ import ( "testing" log "cosmossdk.io/log" - dbm "github.com/cosmos/cosmos-db" "github.com/stretchr/testify/require" + + dbm "github.com/cosmos/iavl/db" ) func TestIterator_NewIterator_NilTree_Failure(t *testing.T) { diff --git a/migrate_test.go b/migrate_test.go index c75167520..6b46301fe 100644 --- a/migrate_test.go +++ b/migrate_test.go @@ -10,8 +10,9 @@ import ( "testing" "cosmossdk.io/log" - dbm "github.com/cosmos/cosmos-db" "github.com/stretchr/testify/require" + + dbm "github.com/cosmos/iavl/db" ) const ( @@ -50,7 +51,7 @@ func TestLazySet(t *testing.T) { relateDir, err := createLegacyTree(t, dbType, dbDir, legacyVersion) require.NoError(t, err) - db, err := dbm.NewDB("test", dbm.GoLevelDBBackend, relateDir) + db, err := dbm.NewDB("test", "goleveldb", relateDir) require.NoError(t, err) defer func() { @@ -93,7 +94,7 @@ func TestLegacyReferenceNode(t *testing.T) { relateDir, err := createLegacyTree(t, dbType, dbDir, legacyVersion) require.NoError(t, err) - db, err := dbm.NewDB("test", dbm.GoLevelDBBackend, relateDir) + db, err := dbm.NewDB("test", "goleveldb", relateDir) require.NoError(t, err) defer func() { @@ -132,7 +133,7 @@ func TestDeleteVersions(t *testing.T) { relateDir, err := createLegacyTree(t, dbType, dbDir, legacyVersion) require.NoError(t, err) - db, err := dbm.NewDB("test", dbm.GoLevelDBBackend, relateDir) + db, err := dbm.NewDB("test", "goleveldb", relateDir) require.NoError(t, err) defer func() { diff --git a/mock/db_mock.go b/mock/db_mock.go index 30c8358c1..4c4d3d515 100644 --- a/mock/db_mock.go +++ b/mock/db_mock.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/cosmos/cosmos-db (interfaces: DB,Iterator,Batch) +// Source: ./db/types.go // Package mock is a generated GoMock package. package mock @@ -7,7 +7,7 @@ package mock import ( reflect "reflect" - db "github.com/cosmos/cosmos-db" + db "github.com/cosmos/iavl/db" gomock "github.com/golang/mock/gomock" ) @@ -48,34 +48,6 @@ func (mr *MockDBMockRecorder) Close() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockDB)(nil).Close)) } -// Delete mocks base method. -func (m *MockDB) Delete(arg0 []byte) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Delete", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// Delete indicates an expected call of Delete. -func (mr *MockDBMockRecorder) Delete(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockDB)(nil).Delete), arg0) -} - -// DeleteSync mocks base method. -func (m *MockDB) DeleteSync(arg0 []byte) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteSync", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteSync indicates an expected call of DeleteSync. -func (mr *MockDBMockRecorder) DeleteSync(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteSync", reflect.TypeOf((*MockDB)(nil).DeleteSync), arg0) -} - // Get mocks base method. func (m *MockDB) Get(arg0 []byte) ([]byte, error) { m.ctrl.T.Helper() @@ -92,33 +64,33 @@ func (mr *MockDBMockRecorder) Get(arg0 interface{}) *gomock.Call { } // Has mocks base method. -func (m *MockDB) Has(arg0 []byte) (bool, error) { +func (m *MockDB) Has(key []byte) (bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Has", arg0) + ret := m.ctrl.Call(m, "Has", key) ret0, _ := ret[0].(bool) ret1, _ := ret[1].(error) return ret0, ret1 } // Has indicates an expected call of Has. -func (mr *MockDBMockRecorder) Has(arg0 interface{}) *gomock.Call { +func (mr *MockDBMockRecorder) Has(key interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Has", reflect.TypeOf((*MockDB)(nil).Has), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Has", reflect.TypeOf((*MockDB)(nil).Has), key) } // Iterator mocks base method. -func (m *MockDB) Iterator(arg0, arg1 []byte) (db.Iterator, error) { +func (m *MockDB) Iterator(start, end []byte) (db.Iterator, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Iterator", arg0, arg1) + ret := m.ctrl.Call(m, "Iterator", start, end) ret0, _ := ret[0].(db.Iterator) ret1, _ := ret[1].(error) return ret0, ret1 } // Iterator indicates an expected call of Iterator. -func (mr *MockDBMockRecorder) Iterator(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockDBMockRecorder) Iterator(start, end interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Iterator", reflect.TypeOf((*MockDB)(nil).Iterator), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Iterator", reflect.TypeOf((*MockDB)(nil).Iterator), start, end) } // NewBatch mocks base method. @@ -149,75 +121,19 @@ func (mr *MockDBMockRecorder) NewBatchWithSize(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewBatchWithSize", reflect.TypeOf((*MockDB)(nil).NewBatchWithSize), arg0) } -// Print mocks base method. -func (m *MockDB) Print() error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Print") - ret0, _ := ret[0].(error) - return ret0 -} - -// Print indicates an expected call of Print. -func (mr *MockDBMockRecorder) Print() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Print", reflect.TypeOf((*MockDB)(nil).Print)) -} - // ReverseIterator mocks base method. -func (m *MockDB) ReverseIterator(arg0, arg1 []byte) (db.Iterator, error) { +func (m *MockDB) ReverseIterator(start, end []byte) (db.Iterator, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ReverseIterator", arg0, arg1) + ret := m.ctrl.Call(m, "ReverseIterator", start, end) ret0, _ := ret[0].(db.Iterator) ret1, _ := ret[1].(error) return ret0, ret1 } // ReverseIterator indicates an expected call of ReverseIterator. -func (mr *MockDBMockRecorder) ReverseIterator(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReverseIterator", reflect.TypeOf((*MockDB)(nil).ReverseIterator), arg0, arg1) -} - -// Set mocks base method. -func (m *MockDB) Set(arg0, arg1 []byte) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Set", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 -} - -// Set indicates an expected call of Set. -func (mr *MockDBMockRecorder) Set(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockDB)(nil).Set), arg0, arg1) -} - -// SetSync mocks base method. -func (m *MockDB) SetSync(arg0, arg1 []byte) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SetSync", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 -} - -// SetSync indicates an expected call of SetSync. -func (mr *MockDBMockRecorder) SetSync(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetSync", reflect.TypeOf((*MockDB)(nil).SetSync), arg0, arg1) -} - -// Stats mocks base method. -func (m *MockDB) Stats() map[string]string { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Stats") - ret0, _ := ret[0].(map[string]string) - return ret0 -} - -// Stats indicates an expected call of Stats. -func (mr *MockDBMockRecorder) Stats() *gomock.Call { +func (mr *MockDBMockRecorder) ReverseIterator(start, end interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stats", reflect.TypeOf((*MockDB)(nil).Stats)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReverseIterator", reflect.TypeOf((*MockDB)(nil).ReverseIterator), start, end) } // MockIterator is a mock of Iterator interface. @@ -378,17 +294,17 @@ func (mr *MockBatchMockRecorder) Close() *gomock.Call { } // Delete mocks base method. -func (m *MockBatch) Delete(arg0 []byte) error { +func (m *MockBatch) Delete(key []byte) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Delete", arg0) + ret := m.ctrl.Call(m, "Delete", key) ret0, _ := ret[0].(error) return ret0 } // Delete indicates an expected call of Delete. -func (mr *MockBatchMockRecorder) Delete(arg0 interface{}) *gomock.Call { +func (mr *MockBatchMockRecorder) Delete(key interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockBatch)(nil).Delete), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockBatch)(nil).Delete), key) } // GetByteSize mocks base method. @@ -407,17 +323,17 @@ func (mr *MockBatchMockRecorder) GetByteSize() *gomock.Call { } // Set mocks base method. -func (m *MockBatch) Set(arg0, arg1 []byte) error { +func (m *MockBatch) Set(key, value []byte) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Set", arg0, arg1) + ret := m.ctrl.Call(m, "Set", key, value) ret0, _ := ret[0].(error) return ret0 } // Set indicates an expected call of Set. -func (mr *MockBatchMockRecorder) Set(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockBatchMockRecorder) Set(key, value interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockBatch)(nil).Set), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockBatch)(nil).Set), key, value) } // Write mocks base method. diff --git a/mutable_tree.go b/mutable_tree.go index 358a366e1..1649675de 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -8,8 +8,8 @@ import ( "sync" log "cosmossdk.io/log" - dbm "github.com/cosmos/cosmos-db" + dbm "github.com/cosmos/iavl/db" "github.com/cosmos/iavl/fastnode" ibytes "github.com/cosmos/iavl/internal/bytes" ) diff --git a/mutable_tree_test.go b/mutable_tree_test.go index 824c2985b..7475c0882 100644 --- a/mutable_tree_test.go +++ b/mutable_tree_test.go @@ -20,7 +20,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - db "github.com/cosmos/cosmos-db" + dbm "github.com/cosmos/iavl/db" ) var ( @@ -34,7 +34,7 @@ var ( ) func setupMutableTree(skipFastStorageUpgrade bool) *MutableTree { - memDB := db.NewMemDB() + memDB := dbm.NewMemDB() tree := NewMutableTree(memDB, 0, skipFastStorageUpgrade, log.NewNopLogger()) return tree } @@ -83,9 +83,8 @@ func TestIteratorConcurrency(t *testing.T) { }(i, j) } itr, _ := tree.Iterator(nil, nil, true) - for ; itr.Valid(); itr.Next() { - // do nothing - } + for ; itr.Valid(); itr.Next() { //nolint:revive + } // do nothing } wg.Wait() } @@ -107,9 +106,8 @@ func TestNewIteratorConcurrency(t *testing.T) { require.NoError(t, err) }(i, j) } - for ; it.Valid(); it.Next() { - // do nothing - } + for ; it.Valid(); it.Next() { //nolint:revive + } // do nothing wg.Wait() } } @@ -258,7 +256,7 @@ func TestMutableTree_LoadVersion_Empty(t *testing.T) { } func TestMutableTree_InitialVersion(t *testing.T) { - memDB := db.NewMemDB() + memDB := dbm.NewMemDB() tree := NewMutableTree(memDB, 0, false, log.NewNopLogger(), InitialVersionOption(9)) _, err := tree.Set([]byte("a"), []byte{0x01}) @@ -309,11 +307,10 @@ func TestMutableTree_SetInitialVersion(t *testing.T) { } func BenchmarkMutableTree_Set(b *testing.B) { - db, err := db.NewDB("test", db.MemDBBackend, "") - require.NoError(b, err) + db := dbm.NewMemDB() t := NewMutableTree(db, 100000, false, log.NewNopLogger()) for i := 0; i < 1000000; i++ { - _, err = t.Set(iavlrand.RandBytes(10), []byte{}) + _, err := t.Set(iavlrand.RandBytes(10), []byte{}) require.NoError(b, err) } b.ReportAllocs() @@ -322,13 +319,13 @@ func BenchmarkMutableTree_Set(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - _, err = t.Set(iavlrand.RandBytes(10), []byte{}) + _, err := t.Set(iavlrand.RandBytes(10), []byte{}) require.NoError(b, err) } } func prepareTree(t *testing.T) *MutableTree { - mdb := db.NewMemDB() + mdb := dbm.NewMemDB() tree := NewMutableTree(mdb, 1000, false, log.NewNopLogger()) for i := 0; i < 100; i++ { _, err := tree.Set([]byte{byte(i)}, []byte("a")) @@ -399,7 +396,7 @@ func TestMutableTree_DeleteVersion(t *testing.T) { } func TestMutableTree_LazyLoadVersionWithEmptyTree(t *testing.T) { - mdb := db.NewMemDB() + mdb := dbm.NewMemDB() tree := NewMutableTree(mdb, 1000, false, log.NewNopLogger()) _, v1, err := tree.SaveVersion() require.NoError(t, err) @@ -418,7 +415,7 @@ func TestMutableTree_LazyLoadVersionWithEmptyTree(t *testing.T) { } func TestMutableTree_SetSimple(t *testing.T) { - mdb := db.NewMemDB() + mdb := dbm.NewMemDB() tree := NewMutableTree(mdb, 0, false, log.NewNopLogger()) const testKey1 = "a" @@ -589,7 +586,7 @@ func TestMutableTree_SetRemoveSet(t *testing.T) { } func TestMutableTree_FastNodeIntegration(t *testing.T) { - mdb := db.NewMemDB() + mdb := dbm.NewMemDB() tree := NewMutableTree(mdb, 1000, false, log.NewNopLogger()) const key1 = "a" @@ -720,7 +717,7 @@ func TestIterator_MutableTree_Invalid(t *testing.T) { func TestUpgradeStorageToFast_LatestVersion_Success(t *testing.T) { // Setup - db := db.NewMemDB() + db := dbm.NewMemDB() tree := NewMutableTree(db, 1000, false, log.NewNopLogger()) // Default version when storage key does not exist in the db @@ -750,7 +747,7 @@ func TestUpgradeStorageToFast_LatestVersion_Success(t *testing.T) { func TestUpgradeStorageToFast_AlreadyUpgraded_Success(t *testing.T) { // Setup - db := db.NewMemDB() + db := dbm.NewMemDB() tree := NewMutableTree(db, 1000, false, log.NewNopLogger()) // Default version when storage key does not exist in the db @@ -1175,7 +1172,8 @@ func TestUpgradeStorageToFast_Delete_Stale_Success(t *testing.T) { valStale := "val_stale" addStaleKey := func(ndb *nodeDB, staleCount int) { - keyPrefix := "key" + keyPrefix := "key_prefix" + b := ndb.db.NewBatch() for i := 0; i < staleCount; i++ { key := fmt.Sprintf("%s_%d", keyPrefix, i) @@ -1184,9 +1182,10 @@ func TestUpgradeStorageToFast_Delete_Stale_Success(t *testing.T) { buf.Grow(node.EncodedSize()) err := node.WriteBytes(&buf) require.NoError(t, err) - err = ndb.db.Set(ndb.fastNodeKey([]byte(key)), buf.Bytes()) + err = b.Set(ndb.fastNodeKey([]byte(key)), buf.Bytes()) require.NoError(t, err) } + require.NoError(t, b.Write()) } type fields struct { nodeCount int @@ -1224,7 +1223,7 @@ func TestUpgradeStorageToFast_Delete_Stale_Success(t *testing.T) { } func setupTreeAndMirror(t *testing.T, numEntries int, skipFastStorageUpgrade bool) (*MutableTree, [][]string) { - db := db.NewMemDB() + db := dbm.NewMemDB() tree := NewMutableTree(db, 0, skipFastStorageUpgrade, log.NewNopLogger()) @@ -1417,7 +1416,7 @@ func TestNoFastStorageUpgrade_Integration_SaveVersion_Load_Iterate_Success(t *te // TestMutableTree_InitialVersion_FirstVersion demonstrate the un-intuitive behavior, // when InitialVersion is set the nodes created in the first version are not assigned with expected version number. func TestMutableTree_InitialVersion_FirstVersion(t *testing.T) { - db := db.NewMemDB() + db := dbm.NewMemDB() initialVersion := int64(1000) tree := NewMutableTree(db, 0, true, log.NewNopLogger(), InitialVersionOption(uint64(initialVersion))) diff --git a/nodedb.go b/nodedb.go index 0dfe45464..269aae7da 100644 --- a/nodedb.go +++ b/nodedb.go @@ -13,10 +13,11 @@ import ( "time" "cosmossdk.io/log" - dbm "github.com/cosmos/cosmos-db" "github.com/cosmos/iavl/cache" + dbm "github.com/cosmos/iavl/db" "github.com/cosmos/iavl/fastnode" + ibytes "github.com/cosmos/iavl/internal/bytes" "github.com/cosmos/iavl/keyformat" ) @@ -326,21 +327,7 @@ func (ndb *nodeDB) saveFastNodeUnlocked(node *fastnode.Node, shouldAddToCache bo // Has checks if a node key exists in the database. func (ndb *nodeDB) Has(nk []byte) (bool, error) { - key := ndb.nodeKey(nk) - - if ldb, ok := ndb.db.(*dbm.GoLevelDB); ok { - exists, err := ldb.DB().Has(key, nil) - if err != nil { - return false, err - } - return exists, nil - } - value, err := ndb.db.Get(key) - if err != nil { - return false, err - } - - return value != nil, nil + return ndb.db.Has(ndb.nodeKey(nk)) } // resetBatch reset the db batch, keep low memory used @@ -427,6 +414,7 @@ var ( // deleteLegacyVersions deletes all legacy versions from disk. func (ndb *nodeDB) deleteLegacyVersions() error { +<<<<<<< HEAD isDeletingLegacyVersionsMutex.Lock() if isDeletingLegacyVersions { isDeletingLegacyVersionsMutex.Unlock() @@ -517,6 +505,10 @@ func (ndb *nodeDB) deleteLegacyVersions() error { // deleteOrphans cleans all legacy orphans from the nodeDB. func (ndb *nodeDB) deleteOrphans() error { itr, err := dbm.IteratePrefix(ndb.db, legacyOrphanKeyFormat.Key()) +======= + // Check if we have a legacy version + itr, err := ndb.getPrefixIterator(legacyRootKeyFormat.Key()) +>>>>>>> 11ba496 (feat: decouple cosmos-db (#874)) if err != nil { return err } @@ -686,7 +678,7 @@ func (ndb *nodeDB) getFirstVersion() (int64, error) { } // Check if we have a legacy version - itr, err := dbm.IteratePrefix(ndb.db, legacyRootKeyFormat.Key()) + itr, err := ndb.getPrefixIterator(legacyRootKeyFormat.Key()) if err != nil { return 0, err } @@ -880,7 +872,7 @@ func (ndb *nodeDB) traverseRange(start []byte, end []byte, fn func(k, v []byte) // Traverse all keys with a certain prefix. Return error if any, nil otherwise func (ndb *nodeDB) traversePrefix(prefix []byte, fn func(k, v []byte) error) error { - itr, err := dbm.IteratePrefix(ndb.db, prefix) + itr, err := ndb.getPrefixIterator(prefix) if err != nil { return err } @@ -895,6 +887,20 @@ func (ndb *nodeDB) traversePrefix(prefix []byte, fn func(k, v []byte) error) err return nil } +// Get the iterator for a given prefix. +func (ndb *nodeDB) getPrefixIterator(prefix []byte) (dbm.Iterator, error) { + var start, end []byte + if len(prefix) == 0 { + start = nil + end = nil + } else { + start = ibytes.Cp(prefix) + end = ibytes.CpIncr(prefix) + } + + return ndb.db.Iterator(start, end) +} + // Get iterator for fast prefix and error, if any func (ndb *nodeDB) getFastIterator(start, end []byte, ascending bool) (dbm.Iterator, error) { var startFormatted, endFormatted []byte diff --git a/nodedb_test.go b/nodedb_test.go index 8d15b09a7..272a8180c 100644 --- a/nodedb_test.go +++ b/nodedb_test.go @@ -8,10 +8,10 @@ import ( "time" log "cosmossdk.io/log" - db "github.com/cosmos/cosmos-db" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + dbm "github.com/cosmos/iavl/db" "github.com/cosmos/iavl/mock" ) @@ -83,7 +83,7 @@ func TestNewNoDbStorage_DoesNotExist_DefaultSet(t *testing.T) { func TestSetStorageVersion_Success(t *testing.T) { const expectedVersion = fastStorageVersionValue - db := db.NewMemDB() + db := dbm.NewMemDB() ndb := newNodeDB(db, 0, DefaultOptions(), log.NewNopLogger()) require.Equal(t, defaultStorageVersionValue, ndb.getStorageVersion()) @@ -144,7 +144,7 @@ func TestSetStorageVersion_InvalidVersionFailure_OldKept(t *testing.T) { } func TestSetStorageVersion_FastVersionFirst_VersionAppended(t *testing.T) { - db := db.NewMemDB() + db := dbm.NewMemDB() ndb := newNodeDB(db, 0, DefaultOptions(), log.NewNopLogger()) ndb.storageVersion = fastStorageVersionValue ndb.latestVersion = 100 @@ -155,7 +155,7 @@ func TestSetStorageVersion_FastVersionFirst_VersionAppended(t *testing.T) { } func TestSetStorageVersion_FastVersionSecond_VersionAppended(t *testing.T) { - db := db.NewMemDB() + db := dbm.NewMemDB() ndb := newNodeDB(db, 0, DefaultOptions(), log.NewNopLogger()) ndb.latestVersion = 100 @@ -169,7 +169,7 @@ func TestSetStorageVersion_FastVersionSecond_VersionAppended(t *testing.T) { } func TestSetStorageVersion_SameVersionTwice(t *testing.T) { - db := db.NewMemDB() + db := dbm.NewMemDB() ndb := newNodeDB(db, 0, DefaultOptions(), log.NewNopLogger()) ndb.latestVersion = 100 @@ -189,7 +189,7 @@ func TestSetStorageVersion_SameVersionTwice(t *testing.T) { // Test case where version is incorrect and has some extra garbage at the end func TestShouldForceFastStorageUpdate_DefaultVersion_True(t *testing.T) { - db := db.NewMemDB() + db := dbm.NewMemDB() ndb := newNodeDB(db, 0, DefaultOptions(), log.NewNopLogger()) ndb.storageVersion = defaultStorageVersionValue ndb.latestVersion = 100 @@ -200,7 +200,7 @@ func TestShouldForceFastStorageUpdate_DefaultVersion_True(t *testing.T) { } func TestShouldForceFastStorageUpdate_FastVersion_Greater_True(t *testing.T) { - db := db.NewMemDB() + db := dbm.NewMemDB() ndb := newNodeDB(db, 0, DefaultOptions(), log.NewNopLogger()) ndb.latestVersion = 100 ndb.storageVersion = fastStorageVersionValue + fastStorageVersionDelimiter + strconv.Itoa(int(ndb.latestVersion+1)) @@ -211,7 +211,7 @@ func TestShouldForceFastStorageUpdate_FastVersion_Greater_True(t *testing.T) { } func TestShouldForceFastStorageUpdate_FastVersion_Smaller_True(t *testing.T) { - db := db.NewMemDB() + db := dbm.NewMemDB() ndb := newNodeDB(db, 0, DefaultOptions(), log.NewNopLogger()) ndb.latestVersion = 100 ndb.storageVersion = fastStorageVersionValue + fastStorageVersionDelimiter + strconv.Itoa(int(ndb.latestVersion-1)) @@ -222,7 +222,7 @@ func TestShouldForceFastStorageUpdate_FastVersion_Smaller_True(t *testing.T) { } func TestShouldForceFastStorageUpdate_FastVersion_Match_False(t *testing.T) { - db := db.NewMemDB() + db := dbm.NewMemDB() ndb := newNodeDB(db, 0, DefaultOptions(), log.NewNopLogger()) ndb.latestVersion = 100 ndb.storageVersion = fastStorageVersionValue + fastStorageVersionDelimiter + strconv.Itoa(int(ndb.latestVersion)) @@ -233,7 +233,7 @@ func TestShouldForceFastStorageUpdate_FastVersion_Match_False(t *testing.T) { } func TestIsFastStorageEnabled_True(t *testing.T) { - db := db.NewMemDB() + db := dbm.NewMemDB() ndb := newNodeDB(db, 0, DefaultOptions(), log.NewNopLogger()) ndb.latestVersion = 100 ndb.storageVersion = fastStorageVersionValue + fastStorageVersionDelimiter + strconv.Itoa(int(ndb.latestVersion)) @@ -242,7 +242,7 @@ func TestIsFastStorageEnabled_True(t *testing.T) { } func TestIsFastStorageEnabled_False(t *testing.T) { - db := db.NewMemDB() + db := dbm.NewMemDB() ndb := newNodeDB(db, 0, DefaultOptions(), log.NewNopLogger()) ndb.latestVersion = 100 ndb.storageVersion = defaultStorageVersionValue @@ -379,7 +379,7 @@ func TestNodeDB_traverseOrphans(t *testing.T) { } func makeAndPopulateMutableTree(tb testing.TB) *MutableTree { - memDB := db.NewMemDB() + memDB := dbm.NewMemDB() tree := NewMutableTree(memDB, 0, false, log.NewNopLogger(), InitialVersionOption(9)) for i := 0; i < 1e4; i++ { @@ -397,7 +397,7 @@ func makeAndPopulateMutableTree(tb testing.TB) *MutableTree { func TestDeleteVersionsFromNoDeadlock(t *testing.T) { const expectedVersion = fastStorageVersionValue - db := db.NewMemDB() + db := dbm.NewMemDB() ndb := newNodeDB(db, 0, DefaultOptions(), log.NewNopLogger()) require.Equal(t, defaultStorageVersionValue, ndb.getStorageVersion()) diff --git a/proof_iavl_test.go b/proof_iavl_test.go index 9327e18fe..50573bf68 100644 --- a/proof_iavl_test.go +++ b/proof_iavl_test.go @@ -5,12 +5,13 @@ import ( "testing" log "cosmossdk.io/log" - db "github.com/cosmos/cosmos-db" "github.com/stretchr/testify/require" + + dbm "github.com/cosmos/iavl/db" ) func TestProofOp(t *testing.T) { - tree := NewMutableTree(db.NewMemDB(), 0, false, log.NewNopLogger()) + tree := NewMutableTree(dbm.NewMemDB(), 0, false, log.NewNopLogger()) keys := []byte{0x0a, 0x11, 0x2e, 0x32, 0x50, 0x72, 0x99, 0xa1, 0xe4, 0xf7} // 10 total. for _, ikey := range keys { key := []byte{ikey} diff --git a/proof_ics23_test.go b/proof_ics23_test.go index 530a674bf..673c20730 100644 --- a/proof_ics23_test.go +++ b/proof_ics23_test.go @@ -11,7 +11,7 @@ import ( ics23 "github.com/cosmos/ics23/go" "github.com/stretchr/testify/require" - db "github.com/cosmos/cosmos-db" + dbm "github.com/cosmos/iavl/db" ) func TestGetMembership(t *testing.T) { @@ -204,7 +204,7 @@ func GetNonKey(allkeys [][]byte, loc Where) []byte { // BuildTree creates random key/values and stores in tree // returns a list of all keys in sorted order func BuildTree(size int, cacheSize int) (itree *MutableTree, keys [][]byte, err error) { - tree := NewMutableTree(db.NewMemDB(), cacheSize, false, log.NewNopLogger()) + tree := NewMutableTree(dbm.NewMemDB(), cacheSize, false, log.NewNopLogger()) // insert lots of info and store the bytes keys = make([][]byte, size) diff --git a/testutils_test.go b/testutils_test.go index 19d31e513..b64a54984 100644 --- a/testutils_test.go +++ b/testutils_test.go @@ -10,9 +10,9 @@ import ( "testing" log "cosmossdk.io/log" - db "github.com/cosmos/cosmos-db" "github.com/stretchr/testify/require" + dbm "github.com/cosmos/iavl/db" "github.com/cosmos/iavl/internal/encoding" iavlrand "github.com/cosmos/iavl/internal/rand" ) @@ -43,7 +43,7 @@ func b2i(bz []byte) int { // Construct a MutableTree func getTestTree(cacheSize int) *MutableTree { - return NewMutableTree(db.NewMemDB(), cacheSize, false, log.NewNopLogger()) + return NewMutableTree(dbm.NewMemDB(), cacheSize, false, log.NewNopLogger()) } // Convenience for a new node @@ -285,7 +285,7 @@ func setupMirrorForIterator(t *testing.T, config *iteratorTestConfig, tree *Muta // assertIterator confirms that the iterator returns the expected values desribed by mirror in the same order. // mirror is a slice containing slices of the form [key, value]. In other words, key at index 0 and value at index 1. -func assertIterator(t *testing.T, itr db.Iterator, mirror [][]string, ascending bool) { +func assertIterator(t *testing.T, itr dbm.Iterator, mirror [][]string, ascending bool) { startIdx, endIdx := 0, len(mirror)-1 increment := 1 mirrorIdx := startIdx @@ -312,12 +312,11 @@ func assertIterator(t *testing.T, itr db.Iterator, mirror [][]string, ascending } func BenchmarkImmutableAvlTreeMemDB(b *testing.B) { - db, err := db.NewDB("test", db.MemDBBackend, "") - require.NoError(b, err) + db := dbm.NewMemDB() benchmarkImmutableAvlTreeWithDB(b, db) } -func benchmarkImmutableAvlTreeWithDB(b *testing.B, db db.DB) { +func benchmarkImmutableAvlTreeWithDB(b *testing.B, db dbm.DB) { defer db.Close() b.StopTimer() diff --git a/tree_random_test.go b/tree_random_test.go index 3ff35c7f1..b5366a87a 100644 --- a/tree_random_test.go +++ b/tree_random_test.go @@ -13,7 +13,7 @@ import ( "cosmossdk.io/log" "github.com/stretchr/testify/require" - db "github.com/cosmos/cosmos-db" + dbm "github.com/cosmos/iavl/db" "github.com/cosmos/iavl/fastnode" ) @@ -68,7 +68,7 @@ func testRandomOperations(t *testing.T, randSeed int64) { r := rand.New(rand.NewSource(randSeed)) // loadTree loads the last persisted version of a tree with random pruning settings. - loadTree := func(levelDB db.DB) (tree *MutableTree, version int64, _ *Options) { //nolint:unparam + loadTree := func(levelDB dbm.DB) (tree *MutableTree, version int64, _ *Options) { //nolint:unparam var err error sync := r.Float64() < syncChance @@ -99,7 +99,7 @@ func testRandomOperations(t *testing.T, randSeed int64) { require.NoError(t, err) defer os.RemoveAll(tempdir) - levelDB, err := db.NewGoLevelDB("leveldb", tempdir, nil) + levelDB, err := dbm.NewDB("test", "goleveldb", tempdir) require.NoError(t, err) tree, version, _ := loadTree(levelDB) diff --git a/tree_test.go b/tree_test.go index 827927189..51ca1a351 100644 --- a/tree_test.go +++ b/tree_test.go @@ -13,10 +13,10 @@ import ( "testing" "cosmossdk.io/log" - db "github.com/cosmos/cosmos-db" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + dbm "github.com/cosmos/iavl/db" iavlrand "github.com/cosmos/iavl/internal/rand" ) @@ -34,9 +34,9 @@ func SetupTest() { flag.Parse() } -func getTestDB() (db.DB, func()) { +func getTestDB() (dbm.DB, func()) { if testLevelDB { - d, err := db.NewGoLevelDB("test", ".", nil) + d, err := dbm.NewDB("test", "goleveldb", ".") if err != nil { panic(err) } @@ -45,7 +45,7 @@ func getTestDB() (db.DB, func()) { os.RemoveAll("./test.db") } } - return db.NewMemDB(), func() {} + return dbm.NewMemDB(), func() {} } func TestVersionedRandomTree(t *testing.T) { @@ -130,7 +130,7 @@ func TestTreeHash(t *testing.T) { require.Len(t, expectHashes, versions, "must have expected hashes for all versions") r := rand.New(rand.NewSource(randSeed)) - tree := NewMutableTree(db.NewMemDB(), 0, false, log.NewNopLogger()) + tree := NewMutableTree(dbm.NewMemDB(), 0, false, log.NewNopLogger()) keys := make([][]byte, 0, versionOps) for i := 0; i < versions; i++ { @@ -708,7 +708,7 @@ func TestVersionedTreeSpecialCase(t *testing.T) { func TestVersionedTreeSpecialCase2(t *testing.T) { require := require.New(t) - d := db.NewMemDB() + d := dbm.NewMemDB() tree := NewMutableTree(d, 100, false, log.NewNopLogger()) tree.Set([]byte("key1"), []byte("val0")) @@ -765,7 +765,7 @@ func TestVersionedTreeSpecialCase3(t *testing.T) { func TestVersionedTreeSaveAndLoad(t *testing.T) { require := require.New(t) - d := db.NewMemDB() + d := dbm.NewMemDB() tree := NewMutableTree(d, 0, false, log.NewNopLogger()) // Loading with an empty root is a no-op. @@ -909,7 +909,7 @@ func TestVersionedCheckpointsSpecialCase3(t *testing.T) { } func TestVersionedCheckpointsSpecialCase4(t *testing.T) { - tree := NewMutableTree(db.NewMemDB(), 0, false, log.NewNopLogger()) + tree := NewMutableTree(dbm.NewMemDB(), 0, false, log.NewNopLogger()) tree.Set([]byte("U"), []byte("XamDUtiJ")) tree.Set([]byte("A"), []byte("UkZBuYIU")) @@ -1028,7 +1028,7 @@ func TestVersionedCheckpointsSpecialCase7(t *testing.T) { func TestVersionedTreeEfficiency(t *testing.T) { require := require.New(t) - tree := NewMutableTree(db.NewMemDB(), 0, false, log.NewNopLogger()) + tree := NewMutableTree(dbm.NewMemDB(), 0, false, log.NewNopLogger()) versions := 20 keysPerVersion := 100 keysAddedPerVersion := map[int]int{} @@ -1156,7 +1156,7 @@ func TestOrphans(t *testing.T) { // Then randomly delete versions other than the first and last until only those two remain // Any remaining orphan nodes should either have fromVersion == firstVersion || toVersion == lastVersion require := require.New(t) - tree := NewMutableTree(db.NewMemDB(), 100, false, log.NewNopLogger()) + tree := NewMutableTree(dbm.NewMemDB(), 100, false, log.NewNopLogger()) NUMVERSIONS := 100 NUMUPDATES := 100 @@ -1306,7 +1306,7 @@ func TestLoadVersion(t *testing.T) { func TestOverwrite(t *testing.T) { require := require.New(t) - mdb := db.NewMemDB() + mdb := dbm.NewMemDB() tree := NewMutableTree(mdb, 0, false, log.NewNopLogger()) // Set one kv pair and save version 1 @@ -1338,7 +1338,7 @@ func TestOverwrite(t *testing.T) { func TestOverwriteEmpty(t *testing.T) { require := require.New(t) - mdb := db.NewMemDB() + mdb := dbm.NewMemDB() tree := NewMutableTree(mdb, 0, false, log.NewNopLogger()) // Save empty version 1 @@ -1372,7 +1372,7 @@ func TestOverwriteEmpty(t *testing.T) { func TestLoadVersionForOverwriting(t *testing.T) { require := require.New(t) - mdb := db.NewMemDB() + mdb := dbm.NewMemDB() tree := NewMutableTree(mdb, 0, false, log.NewNopLogger()) maxLength := 100 @@ -1439,7 +1439,7 @@ func BenchmarkTreeLoadAndDelete(b *testing.B) { numVersions := 5000 numKeysPerVersion := 10 - d, err := db.NewGoLevelDB("bench", ".", nil) + d, err := dbm.NewDB("bench", "goleveldb", ".") if err != nil { panic(err) } @@ -1481,7 +1481,7 @@ func BenchmarkTreeLoadAndDelete(b *testing.B) { func TestLoadVersionForOverwritingCase2(t *testing.T) { require := require.New(t) - tree := NewMutableTree(db.NewMemDB(), 0, false, log.NewNopLogger()) + tree := NewMutableTree(dbm.NewMemDB(), 0, false, log.NewNopLogger()) for i := byte(0); i < 20; i++ { tree.Set([]byte{i}, []byte{i}) @@ -1543,7 +1543,7 @@ func TestLoadVersionForOverwritingCase2(t *testing.T) { func TestLoadVersionForOverwritingCase3(t *testing.T) { require := require.New(t) - tree := NewMutableTree(db.NewMemDB(), 0, false, log.NewNopLogger()) + tree := NewMutableTree(dbm.NewMemDB(), 0, false, log.NewNopLogger()) for i := byte(0); i < 20; i++ { tree.Set([]byte{i}, []byte{i}) @@ -1668,7 +1668,7 @@ func TestGetWithIndex_ImmutableTree(t *testing.T) { } func Benchmark_GetWithIndex(b *testing.B) { - db, err := db.NewDB("test", db.MemDBBackend, "") + db, err := dbm.NewDB("test", "memdb", "") require.NoError(b, err) const numKeyVals = 100000 @@ -1719,7 +1719,7 @@ func Benchmark_GetWithIndex(b *testing.B) { } func Benchmark_GetByIndex(b *testing.B) { - db, err := db.NewDB("test", db.MemDBBackend, "") + db, err := dbm.NewDB("test", "memdb", "") require.NoError(b, err) const numKeyVals = 100000 @@ -1796,7 +1796,7 @@ func TestNodeCacheStatisic(t *testing.T) { tc := tc t.Run(name, func(sub *testing.T) { stat := &Statistics{} - db, err := db.NewDB("test", db.MemDBBackend, "") + db, err := dbm.NewDB("test", "memdb", "") require.NoError(t, err) mt := NewMutableTree(db, tc.cacheSize, false, log.NewNopLogger(), StatOption(stat)) From 3f26d7c5b3eac3ac67d1341f1ab46497d1044fbc Mon Sep 17 00:00:00 2001 From: Cool Developer Date: Wed, 21 Feb 2024 07:46:46 -0500 Subject: [PATCH 2/3] fix --- .github/workflows/lint.yml | 6 ------ batch.go | 4 ---- nodedb.go | 10 +--------- 3 files changed, 1 insertion(+), 19 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b298a8d00..b6e18f3f9 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -17,14 +17,8 @@ jobs: - name: 🐿 Setup Golang uses: actions/setup-go@v4 with: -<<<<<<< HEAD go-version: 1.21 - - name: golangci-lint - run: make lint -======= - go-version: "^1.20.0" - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: version: v1.55.2 ->>>>>>> 11ba496 (feat: decouple cosmos-db (#874)) diff --git a/batch.go b/batch.go index 50f2a91aa..c67306393 100644 --- a/batch.go +++ b/batch.go @@ -1,13 +1,9 @@ package iavl import ( -<<<<<<< HEAD "sync" - dbm "github.com/cosmos/cosmos-db" -======= dbm "github.com/cosmos/iavl/db" ->>>>>>> 11ba496 (feat: decouple cosmos-db (#874)) ) // BatchWithFlusher is a wrapper diff --git a/nodedb.go b/nodedb.go index 269aae7da..17f279425 100644 --- a/nodedb.go +++ b/nodedb.go @@ -62,9 +62,6 @@ var ( // All legacy node keys are prefixed with the byte 'n'. legacyNodeKeyFormat = keyformat.NewFastPrefixFormatter('n', hashSize) // n - // All legacy orphan keys are prefixed with the byte 'o'. - legacyOrphanKeyFormat = keyformat.NewKeyFormat('o', int64Size, int64Size, hashSize) // o - // All legacy root keys are prefixed with the byte 'r'. legacyRootKeyFormat = keyformat.NewKeyFormat('r', int64Size) // r ) @@ -414,7 +411,6 @@ var ( // deleteLegacyVersions deletes all legacy versions from disk. func (ndb *nodeDB) deleteLegacyVersions() error { -<<<<<<< HEAD isDeletingLegacyVersionsMutex.Lock() if isDeletingLegacyVersions { isDeletingLegacyVersionsMutex.Unlock() @@ -431,7 +427,7 @@ func (ndb *nodeDB) deleteLegacyVersions() error { }() // Check if we have a legacy version - itr, err := dbm.IteratePrefix(ndb.db, legacyRootKeyFormat.Key()) + itr, err := ndb.getPrefixIterator(legacyRootKeyFormat.Key()) if err != nil { ndb.logger.Error(err.Error()) return @@ -504,11 +500,7 @@ func (ndb *nodeDB) deleteLegacyVersions() error { // deleteOrphans cleans all legacy orphans from the nodeDB. func (ndb *nodeDB) deleteOrphans() error { - itr, err := dbm.IteratePrefix(ndb.db, legacyOrphanKeyFormat.Key()) -======= - // Check if we have a legacy version itr, err := ndb.getPrefixIterator(legacyRootKeyFormat.Key()) ->>>>>>> 11ba496 (feat: decouple cosmos-db (#874)) if err != nil { return err } From eb0e03063440d86258d5db5fb19c9fcbdb1058db Mon Sep 17 00:00:00 2001 From: Cool Developer Date: Wed, 21 Feb 2024 07:56:56 -0500 Subject: [PATCH 3/3] lint --- .github/workflows/lint.yml | 4 +--- .golangci.yml | 1 - basic_test.go | 2 +- tree_dotgraph_test.go | 2 +- tree_random_test.go | 2 +- tree_test.go | 10 +++++----- 6 files changed, 9 insertions(+), 12 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b6e18f3f9..1e03feecc 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -19,6 +19,4 @@ jobs: with: go-version: 1.21 - name: golangci-lint - uses: golangci/golangci-lint-action@v3 - with: - version: v1.55.2 + run: make lint diff --git a/.golangci.yml b/.golangci.yml index 57ecead80..816d9c4a9 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -7,7 +7,6 @@ linters: disable-all: true enable: - bodyclose - - depguard - dogsled - errcheck - exportloopref diff --git a/basic_test.go b/basic_test.go index e1e73cdd7..88134d65d 100644 --- a/basic_test.go +++ b/basic_test.go @@ -237,7 +237,7 @@ func TestUnit(t *testing.T) { expectRemove(t11, 3, "((1 2) (4 5))") } -func TestRemove(t *testing.T) { +func TestRemove(_ *testing.T) { keyLen, dataLen := 16, 40 size := 10000 diff --git a/tree_dotgraph_test.go b/tree_dotgraph_test.go index 41029307d..41efdfcbe 100644 --- a/tree_dotgraph_test.go +++ b/tree_dotgraph_test.go @@ -5,7 +5,7 @@ import ( "testing" ) -func TestWriteDOTGraph(t *testing.T) { +func TestWriteDOTGraph(_ *testing.T) { tree := getTestTree(0) for _, ikey := range []byte{ 0x0a, 0x11, 0x2e, 0x32, 0x50, 0x72, 0x99, 0xa1, 0xe4, 0xf7, diff --git a/tree_random_test.go b/tree_random_test.go index b5366a87a..08739d432 100644 --- a/tree_random_test.go +++ b/tree_random_test.go @@ -300,7 +300,7 @@ func assertOrphans(t *testing.T, tree *MutableTree, expected int) { } // Checks that a version is the maximum mirrored version. -func assertMaxVersion(t *testing.T, tree *MutableTree, version int64, mirrors map[int64]map[string]string) { //nolint:unparam +func assertMaxVersion(t *testing.T, _ *MutableTree, version int64, mirrors map[int64]map[string]string) { max := int64(0) for v := range mirrors { if v > max { diff --git a/tree_test.go b/tree_test.go index 51ca1a351..cf71ff2cd 100644 --- a/tree_test.go +++ b/tree_test.go @@ -868,7 +868,7 @@ func TestVersionedCheckpointsSpecialCase(t *testing.T) { require.Equal([]byte("val1"), val) } -func TestVersionedCheckpointsSpecialCase2(t *testing.T) { +func TestVersionedCheckpointsSpecialCase2(_ *testing.T) { tree := getTestTree(0) tree.Set([]byte("U"), []byte("XamDUtiJ")) @@ -888,7 +888,7 @@ func TestVersionedCheckpointsSpecialCase2(t *testing.T) { tree.DeleteVersionsTo(2) } -func TestVersionedCheckpointsSpecialCase3(t *testing.T) { +func TestVersionedCheckpointsSpecialCase3(_ *testing.T) { tree := getTestTree(0) tree.Set([]byte("n"), []byte("2wUCUs8q")) @@ -944,7 +944,7 @@ func TestVersionedCheckpointsSpecialCase4(t *testing.T) { require.Nil(t, val) } -func TestVersionedCheckpointsSpecialCase5(t *testing.T) { +func TestVersionedCheckpointsSpecialCase5(_ *testing.T) { tree := getTestTree(0) tree.Set([]byte("R"), []byte("ygZlIzeW")) @@ -961,7 +961,7 @@ func TestVersionedCheckpointsSpecialCase5(t *testing.T) { tree.GetVersioned([]byte("R"), 2) } -func TestVersionedCheckpointsSpecialCase6(t *testing.T) { +func TestVersionedCheckpointsSpecialCase6(_ *testing.T) { tree := getTestTree(0) tree.Set([]byte("Y"), []byte("MW79JQeV")) @@ -993,7 +993,7 @@ func TestVersionedCheckpointsSpecialCase6(t *testing.T) { tree.GetVersioned([]byte("4"), 1) } -func TestVersionedCheckpointsSpecialCase7(t *testing.T) { +func TestVersionedCheckpointsSpecialCase7(_ *testing.T) { tree := getTestTree(100) tree.Set([]byte("n"), []byte("OtqD3nyn"))