From 0b6f4eb579cf46ab1ea861abddca25b81cc0d94c Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Mon, 2 Jul 2018 18:34:19 -0700 Subject: [PATCH 01/10] Jae/rangeprooffix (#75) * VerifyItem takes no index; Return keys/values in range; Fix * Bump version * 0.9.1 -- compute hash from rangeproof * Require Verify() before Verify*() * Review fixes from #75 --- CHANGELOG.md | 19 ++++ basic_test.go | 2 +- proof_path.go | 48 +++++++++- proof_range.go | 232 ++++++++++++++++++++++++++++++++----------------- proof_test.go | 126 ++++++++++++++++----------- tree_test.go | 8 +- version.go | 2 +- 7 files changed, 295 insertions(+), 142 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd3e5e36a..24615000e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog +## 0.9.1 (July 1, 2018) + +IMPROVEMENTS + +- RangeProof.ComputeRootHash() to compute root rather than provide as in Verify(hash) +- RangeProof.Verify\*() first require .Verify(root), which memoizes + +## 0.9.0 (July 1, 2018) + +BREAKING CHANGES + +- RangeProof.VerifyItem doesn't require an index. +- Only return values in range when getting proof. +- Return keys as well. + +BUG FIXES + +- traversal bugs in traverseRange. + ## 0.8.1 *July 1st, 2018* diff --git a/basic_test.go b/basic_test.go index 6e05215ee..0710afe36 100644 --- a/basic_test.go +++ b/basic_test.go @@ -450,7 +450,7 @@ func TestTreeProof(t *testing.T) { assert.Equal(t, key, value) err := proof.Verify(root) assert.NoError(t, err, "#### %v", proof.String()) - err = proof.VerifyItem(0, key, key) + err = proof.VerifyItem(key, key) assert.NoError(t, err, "#### %v", proof.String()) } } diff --git a/proof_path.go b/proof_path.go index 29b9ebc76..dbde9fa10 100644 --- a/proof_path.go +++ b/proof_path.go @@ -28,8 +28,19 @@ func (pwl pathWithLeaf) StringIndented(indent string) string { indent) } +// `verify` checks that the leaf node's hash + the inner nodes merkle-izes to +// the given root. If it returns an error, it means the leafHash or the +// PathToLeaf is incorrect. func (pwl pathWithLeaf) verify(root []byte) cmn.Error { - return pwl.Path.verify(pwl.Leaf.Hash(), root) + leafHash := pwl.Leaf.Hash() + return pwl.Path.verify(leafHash, root) +} + +// `computeRootHash` computes the root hash with leaf node. +// Does not verify the root hash. +func (pwl pathWithLeaf) computeRootHash() []byte { + leafHash := pwl.Leaf.Hash() + return pwl.Path.computeRootHash(leafHash) } //---------------------------------------- @@ -62,9 +73,9 @@ func (pl PathToLeaf) StringIndented(indent string) string { indent) } -// verify checks that the leaf node's hash + the inner nodes merkle-izes to the -// given root. If it returns an error, it means the leafHash or the PathToLeaf -// is incorrect. +// `verify` checks that the leaf node's hash + the inner nodes merkle-izes to +// the given root. If it returns an error, it means the leafHash or the +// PathToLeaf is incorrect. func (pl PathToLeaf) verify(leafHash []byte, root []byte) cmn.Error { hash := leafHash for i := len(pl) - 1; i >= 0; i-- { @@ -77,6 +88,17 @@ func (pl PathToLeaf) verify(leafHash []byte, root []byte) cmn.Error { return nil } +// `computeRootHash` computes the root hash assuming some leaf hash. +// Does not verify the root hash. +func (pl PathToLeaf) computeRootHash(leafHash []byte) []byte { + hash := leafHash + for i := len(pl) - 1; i >= 0; i-- { + pin := pl[i] + hash = pin.Hash(hash) + } + return hash +} + func (pl PathToLeaf) isLeftmost() bool { for _, node := range pl { if len(node.Left) > 0 { @@ -125,3 +147,21 @@ func (pl PathToLeaf) isLeftAdjacentTo(pl2 PathToLeaf) bool { return pl.isRightmost() && pl2.isLeftmost() } + +// returns -1 if invalid. +func (pl PathToLeaf) Index() (idx int64) { + for i, node := range pl { + if node.Left == nil { + continue + } else if node.Right == nil { + if i < len(pl)-1 { + idx += node.Size - pl[i+1].Size + } else { + idx += node.Size - 1 + } + } else { + return -1 + } + } + return idx +} diff --git a/proof_range.go b/proof_range.go index 7daa80006..ad4f4d915 100644 --- a/proof_range.go +++ b/proof_range.go @@ -3,6 +3,7 @@ package iavl import ( "bytes" "fmt" + "sort" "strings" "github.com/tendermint/iavl/sha256truncated" @@ -12,16 +13,39 @@ import ( type RangeProof struct { // You don't need the right path because // it can be derived from what we have. - RootHash cmn.HexBytes `json:"root_hash"` LeftPath PathToLeaf `json:"left_path"` InnerNodes []PathToLeaf `json:"inner_nodes"` Leaves []proofLeafNode `json:"leaves"` - // temporary - treeEnd int // 0 if not set, 1 if true, -1 if false. + + // memoize + rootVerified bool + rootHash []byte // valid iff rootVerified is true + treeEnd bool // valid iff rootVerified is true + +} + +// Keys returns all the keys in the RangeProof. NOTE: The keys here may +// include more keys than provided by tree.GetRangeWithProof or +// VersionedTree.GetVersionedRangeWithProof. The keys returned there are only +// in the provided [startKey,endKey){limit} range. The keys returned here may +// include extra keys, such as: +// - the key before startKey if startKey is provided and doesn't exist; +// - the key after a queried key with tree.GetWithProof, when the key is absent. +func (proof *RangeProof) Keys() (keys [][]byte) { + if proof == nil { + return nil + } + for _, leaf := range proof.Leaves { + keys = append(keys, leaf.Key) + } + return keys } // String returns a string representation of the proof. func (proof *RangeProof) String() string { + if proof == nil { + return "" + } return proof.StringIndented("") } @@ -35,36 +59,54 @@ func (proof *RangeProof) StringIndented(indent string) string { lstrs = append(lstrs, leaf.StringIndented(indent+" ")) } return fmt.Sprintf(`RangeProof{ -%s RootHash: %X %s LeftPath: %v %s InnerNodes: %s %v %s Leaves: %s %v +%s (rootVerified): %v +%s (rootHash): %X %s (treeEnd): %v %s}`, - indent, proof.RootHash, indent, proof.LeftPath.StringIndented(indent+" "), indent, indent, strings.Join(istrs, "\n"+indent+" "), indent, indent, strings.Join(lstrs, "\n"+indent+" "), + indent, proof.rootVerified, + indent, proof.rootHash, indent, proof.treeEnd, indent) } -// Verify that a leaf is some value. -// Does not assume that the proof itself is value. -// For that, use Verify(root). -func (proof *RangeProof) VerifyItem(i int, key, value []byte) error { +// The index of the first leaf (of the whole tree). +// Returns -1 if the proof is nil. +func (proof *RangeProof) LeftIndex() int64 { + if proof == nil { + return -1 + } + return proof.LeftPath.Index() +} + +// Also see LeftIndex(). +// Verify that a key has some value. +// Does not assume that the proof itself is valid, call Verify() first. +func (proof *RangeProof) VerifyItem(key, value []byte) error { + leaves := proof.Leaves if proof == nil { return cmn.ErrorWrap(ErrInvalidProof, "proof is nil") } - if !bytes.Equal(proof.Leaves[i].Key, key) { - return cmn.ErrorWrap(ErrInvalidProof, "leaf key not same") + if !proof.rootVerified { + return cmn.NewError("must call Verify(root) first.") + } + i := sort.Search(len(leaves), func(i int) bool { + return bytes.Compare(key, leaves[i].Key) <= 0 + }) + if i >= len(leaves) || !bytes.Equal(leaves[i].Key, key) { + return cmn.ErrorWrap(ErrInvalidProof, "leaf key not found in proof") } valueHash := sha256truncated.Hash(value) - if !bytes.Equal(proof.Leaves[i].ValueHash, valueHash) { + if !bytes.Equal(leaves[i].ValueHash, valueHash) { return cmn.ErrorWrap(ErrInvalidProof, "leaf value hash not same") } return nil @@ -77,7 +119,7 @@ func (proof *RangeProof) VerifyAbsence(key []byte) error { if proof == nil { return cmn.ErrorWrap(ErrInvalidProof, "proof is nil") } - if proof.treeEnd == 0 { + if !proof.rootVerified { return cmn.NewError("must call Verify(root) first.") } cmp := bytes.Compare(key, proof.Leaves[0].Key) @@ -116,7 +158,7 @@ func (proof *RangeProof) VerifyAbsence(key []byte) error { } // It's still a valid proof if our last leaf is the rightmost child. - if proof.treeEnd == 1 { + if proof.treeEnd { return nil // OK! } @@ -133,26 +175,53 @@ func (proof *RangeProof) Verify(root []byte) error { if proof == nil { return cmn.ErrorWrap(ErrInvalidProof, "proof is nil") } - treeEnd, err := proof._verify(root) - if err == nil { - if treeEnd { - proof.treeEnd = 1 // memoize - } else { - proof.treeEnd = -1 // memoize + err := proof.verify(root) + return err +} + +func (proof *RangeProof) verify(root []byte) (err error) { + rootHash := proof.rootHash + if rootHash == nil { + derivedHash, err := proof.computeRootHash() + if err != nil { + return err } + rootHash = derivedHash } - return err + if !bytes.Equal(rootHash, root) { + return cmn.ErrorWrap(ErrInvalidRoot, "root hash doesn't match") + } else { + proof.rootVerified = true + } + return nil } -func (proof *RangeProof) _verify(root []byte) (treeEnd bool, err error) { - if !bytes.Equal(proof.RootHash, root) { - return false, cmn.ErrorWrap(ErrInvalidRoot, "root hash doesn't match") +// ComputeRootHash computes the root hash with leaves. +// Returns nil if error or proof is nil. +// Does not verify the root hash. +func (proof *RangeProof) ComputeRootHash() []byte { + if proof == nil { + return nil } + rootHash, _ := proof.computeRootHash() + return rootHash +} + +func (proof *RangeProof) computeRootHash() (rootHash []byte, err error) { + rootHash, treeEnd, err := proof._computeRootHash() + if err == nil { + proof.rootHash = rootHash // memoize + proof.treeEnd = treeEnd // memoize + } + return rootHash, err +} + +func (proof *RangeProof) _computeRootHash() (rootHash []byte, treeEnd bool, err error) { if len(proof.Leaves) == 0 { - return false, cmn.ErrorWrap(ErrInvalidProof, "no leaves") + return nil, false, cmn.ErrorWrap(ErrInvalidProof, "no leaves") } if len(proof.InnerNodes)+1 != len(proof.Leaves) { - return false, cmn.ErrorWrap(ErrInvalidProof, "InnerNodes vs Leaves length mismatch, leaves should be 1 more.") + return nil, false, cmn.ErrorWrap(ErrInvalidProof, "InnerNodes vs Leaves length mismatch, leaves should be 1 more.") } // Start from the left path and prove each leaf. @@ -160,28 +229,27 @@ func (proof *RangeProof) _verify(root []byte) (treeEnd bool, err error) { // shared across recursive calls var leaves = proof.Leaves var innersq = proof.InnerNodes - var VERIFY func(path PathToLeaf, root []byte, rightmost bool) (treeEnd bool, done bool, err error) + var COMPUTEHASH func(path PathToLeaf, rightmost bool) (hash []byte, treeEnd bool, done bool, err error) + // rightmost: is the root a rightmost child of the tree? // treeEnd: true iff the last leaf is the last item of the tree. - // NOTE: root doesn't necessarily mean root of the tree here. - VERIFY = func(path PathToLeaf, root []byte, rightmost bool) (treeEnd bool, done bool, err error) { + // Returns the (possibly intermediate, possibly root) hash. + COMPUTEHASH = func(path PathToLeaf, rightmost bool) (hash []byte, treeEnd bool, done bool, err error) { // Pop next leaf. nleaf, rleaves := leaves[0], leaves[1:] leaves = rleaves - // Verify leaf with path. - if err := (pathWithLeaf{ + // Compute hash. + hash = (pathWithLeaf{ Path: path, Leaf: nleaf, - }).verify(root); err != nil { - return false, false, err.Trace(0, "verifying some path to a leaf") - } + }).computeRootHash() // If we don't have any leaves left, we're done. if len(leaves) == 0 { rightmost = rightmost && path.isRightmost() - return rightmost, true, nil + return hash, rightmost, true, nil } // Prove along path (until we run out of leaves). @@ -203,31 +271,35 @@ func (proof *RangeProof) _verify(root []byte) (treeEnd bool, err error) { innersq = rinnersq // Recursively verify inners against remaining leaves. - treeEnd, done, err := VERIFY(inners, lpath.Right, rightmost && rpath.isRightmost()) + derivedRoot, treeEnd, done, err := COMPUTEHASH(inners, rightmost && rpath.isRightmost()) if err != nil { - return treeEnd, false, cmn.ErrorWrap(err, "recursive VERIFY call") - } else if done { - return treeEnd, true, nil + return nil, treeEnd, false, cmn.ErrorWrap(err, "recursive COMPUTEHASH call") + } + if !bytes.Equal(derivedRoot, lpath.Right) { + return nil, treeEnd, false, cmn.ErrorWrap(ErrInvalidRoot, "intermediate root hash %X doesn't match, got %X", lpath.Right, derivedRoot) + } + if done { + return hash, treeEnd, true, nil } } - // We're not done yet. No error, not done either. Technically if - // rightmost, we know there's an error "left over leaves -- malformed - // proof", but we return that at the top level, below. - return false, false, nil + // We're not done yet (leaves left over). No error, not done either. + // Technically if rightmost, we know there's an error "left over leaves + // -- malformed proof", but we return that at the top level, below. + return hash, false, false, nil } // Verify! path := proof.LeftPath - treeEnd, done, err := VERIFY(path, root, true) + rootHash, treeEnd, done, err := COMPUTEHASH(path, true) if err != nil { - return treeEnd, cmn.ErrorWrap(err, "root VERIFY call") + return nil, treeEnd, cmn.ErrorWrap(err, "root COMPUTEHASH call") } else if !done { - return treeEnd, cmn.ErrorWrap(ErrInvalidProof, "left over leaves -- malformed proof") + return nil, treeEnd, cmn.ErrorWrap(ErrInvalidProof, "left over leaves -- malformed proof") } // Ok! - return treeEnd, nil + return rootHash, treeEnd, nil } /////////////////////////////////////////////////////////////////////////////// @@ -236,13 +308,17 @@ func (proof *RangeProof) _verify(root []byte) (treeEnd bool, err error) { // If keyStart or keyEnd don't exist, the leaf before keyStart // or after keyEnd will also be included, but not be included in values. // If keyEnd-1 exists, no later leaves will be included. +// If keyStart >= keyEnd and both not nil, panics. // Limit is never exceeded. -func (t *Tree) getRangeProof(keyStart, keyEnd []byte, limit int) (proof *RangeProof, values [][]byte, err error) { +func (t *Tree) getRangeProof(keyStart, keyEnd []byte, limit int) (proof *RangeProof, keys, values [][]byte, err error) { + if keyStart != nil && keyEnd != nil && bytes.Compare(keyStart, keyEnd) >= 0 { + panic("if keyStart and keyEnd are present, need keyStart < keyEnd.") + } if limit < 0 { panic("limit must be greater or equal to 0 -- 0 means no limit") } if t.root == nil { - return nil, nil, cmn.ErrorWrap(ErrNilRoot, "") + return nil, nil, nil, cmn.ErrorWrap(ErrNilRoot, "") } t.root.hashWithCount() // Ensure that all hashes are calculated. @@ -250,10 +326,17 @@ func (t *Tree) getRangeProof(keyStart, keyEnd []byte, limit int) (proof *RangePr path, left, err := t.root.PathToLeaf(t, keyStart) if err != nil { // Key doesn't exist, but instead we got the prev leaf (or the - // first leaf), which provides proof of absence). + // first or last leaf), which provides proof of absence). err = nil } - values = append(values, left.value) + startOK := keyStart == nil || bytes.Compare(keyStart, left.key) <= 0 + endOK := keyEnd == nil || bytes.Compare(left.key, keyEnd) < 0 + // If left.key is in range, add it to key/values. + if startOK && endOK { + keys = append(keys, left.key) // == keyStart + values = append(values, left.value) + } + // Either way, add to proof leaves. var leaves = []proofLeafNode{proofLeafNode{ Key: left.key, ValueHash: sha256truncated.Hash(left.value), @@ -270,18 +353,9 @@ func (t *Tree) getRangeProof(keyStart, keyEnd []byte, limit int) (proof *RangePr } if _stop { return &RangeProof{ - RootHash: t.root.hash, LeftPath: path, Leaves: leaves, - }, values, nil - } - - if keyEnd != nil && bytes.Compare(cpIncr(left.key), keyEnd) >= 0 { - return &RangeProof{ - RootHash: t.root.hash, - LeftPath: path, - Leaves: leaves, - }, values, nil + }, keys, values, nil } // Get the key after left.key to iterate from. @@ -295,7 +369,7 @@ func (t *Tree) getRangeProof(keyStart, keyEnd []byte, limit int) (proof *RangePr var lastDepth uint8 = 0 var leafCount = 1 // from left above. var pathCount = 0 - // var values [][]byte defined as function outs. + // var keys, values [][]byte defined as function outs. t.root.traverseInRange(t, afterLeft, nil, true, false, 0, func(node *Node, depth uint8) (stop bool) { @@ -331,14 +405,20 @@ func (t *Tree) getRangeProof(keyStart, keyEnd []byte, limit int) (proof *RangePr ValueHash: sha256truncated.Hash(node.value), Version: node.version, }) - // Append value to values. - values = append(values, node.value) leafCount += 1 // Maybe terminate because we found enough leaves. if limit > 0 && limit <= leafCount { return true } - // Maybe terminate because we've found keyEnd-1 or after. + // Terminate if we've found keyEnd or after. + if keyEnd != nil && bytes.Compare(node.key, keyEnd) >= 0 { + return true + } + // Value is in range, append to keys and values. + keys = append(keys, node.key) + values = append(values, node.value) + // Terminate if we've found keyEnd-1 or after. + // We don't want to fetch any leaves for it. if keyEnd != nil && bytes.Compare(cpIncr(node.key), keyEnd) >= 0 { return true } @@ -362,11 +442,10 @@ func (t *Tree) getRangeProof(keyStart, keyEnd []byte, limit int) (proof *RangePr ) return &RangeProof{ - RootHash: t.root.hash, LeftPath: path, InnerNodes: innersq, Leaves: leaves, - }, values, nil + }, keys, values, nil } //---------------------------------------- @@ -374,7 +453,7 @@ func (t *Tree) getRangeProof(keyStart, keyEnd []byte, limit int) (proof *RangePr // GetWithProof gets the value under the key if it exists, or returns nil. // A proof of existence or absence is returned alongside the value. func (t *Tree) GetWithProof(key []byte) (value []byte, proof *RangeProof, err error) { - proof, values, err := t.getRangeProof(key, cpIncr(key), 2) + proof, _, values, err := t.getRangeProof(key, cpIncr(key), 2) if err == nil { if len(values) > 0 { if !bytes.Equal(proof.Leaves[0].Key, key) { @@ -390,18 +469,13 @@ func (t *Tree) GetWithProof(key []byte) (value []byte, proof *RangeProof, err er } // GetRangeWithProof gets key/value pairs within the specified range and limit. -// To specify a descending range, swap the start and end keys. func (t *Tree) GetRangeWithProof(startKey []byte, endKey []byte, limit int) (keys, values [][]byte, proof *RangeProof, err error) { - proof, values, err = t.getRangeProof(startKey, endKey, limit) - for _, leaf := range proof.Leaves { - keys = append(keys, leaf.Key) - } + proof, keys, values, err = t.getRangeProof(startKey, endKey, limit) return } // GetVersionedWithProof gets the value under the key at the specified version -// if it exists, or returns nil. A proof of existence or absence is returned -// alongside the value. +// if it exists, or returns nil. func (tree *VersionedTree) GetVersionedWithProof(key []byte, version int64) ([]byte, *RangeProof, error) { if t, ok := tree.versions[version]; ok { return t.GetWithProof(key) @@ -410,10 +484,10 @@ func (tree *VersionedTree) GetVersionedWithProof(key []byte, version int64) ([]b } // GetVersionedRangeWithProof gets key/value pairs within the specified range -// and limit. To specify a descending range, swap the start and end keys. -// -// Returns a list of values, a list of keys, and a proof. -func (tree *VersionedTree) GetVersionedRangeWithProof(startKey, endKey []byte, limit int, version int64) ([][]byte, [][]byte, *RangeProof, error) { +// and limit. +func (tree *VersionedTree) GetVersionedRangeWithProof(startKey, endKey []byte, limit int, version int64) ( + keys, values [][]byte, proof *RangeProof, err error) { + if t, ok := tree.versions[version]; ok { return t.GetRangeWithProof(startKey, endKey, limit) } diff --git a/proof_test.go b/proof_test.go index 43c0ce53f..77f423464 100644 --- a/proof_test.go +++ b/proof_test.go @@ -25,9 +25,11 @@ func TestTreeGetWithProof(t *testing.T) { require.NoError(err) require.NotEmpty(val) require.NotNil(proof) + err = proof.VerifyItem(key, val) + require.Error(err, "%+v", err) // Verifying item before calling Verify(root) err = proof.Verify(root) require.NoError(err, "%+v", err) - err = proof.VerifyItem(0, key, val) + err = proof.VerifyItem(key, val) require.NoError(err, "%+v", err) key = []byte{0x1} @@ -35,6 +37,8 @@ func TestTreeGetWithProof(t *testing.T) { require.NoError(err) require.Empty(val) require.NotNil(proof) + err = proof.VerifyAbsence(key) + require.Error(err, "%+v", err) // Verifying absence before calling Verify(root) err = proof.Verify(root) require.NoError(err, "%+v", err) err = proof.VerifyAbsence(key) @@ -46,52 +50,50 @@ func TestTreeKeyExistsProof(t *testing.T) { root := tree.Hash() // should get false for proof with nil root - proof, _, err := tree.getRangeProof([]byte("foo"), nil, 1) + proof, _, _, err := tree.getRangeProof([]byte("foo"), nil, 1) assert.NotNil(t, err) assert.NotNil(t, proof.Verify(root)) // insert lots of info and store the bytes - keys := make([][]byte, 200) + allkeys := make([][]byte, 200) for i := 0; i < 200; i++ { key := randstr(20) value := "value_for_" + key tree.Set([]byte(key), []byte(value)) - keys[i] = []byte(key) + allkeys[i] = []byte(key) } - sortByteSlices(keys) // Sort keys + sortByteSlices(allkeys) // Sort all keys root = tree.Hash() // query random key fails - proof, _, err = tree.getRangeProof([]byte("foo"), nil, 2) + proof, _, _, err = tree.getRangeProof([]byte("foo"), nil, 2) assert.Nil(t, err) assert.Nil(t, proof.Verify(root)) assert.Nil(t, proof.VerifyAbsence([]byte("foo")), proof.String()) // query min key fails - proof, _, err = tree.getRangeProof([]byte{0x00}, []byte{0x00}, 2) + proof, _, _, err = tree.getRangeProof([]byte{0x00}, []byte{0x01}, 2) assert.Nil(t, err) assert.Nil(t, proof.Verify(root)) assert.Nil(t, proof.VerifyAbsence([]byte{0x00})) // valid proof for real keys - for i, key := range keys { - var values [][]byte - proof, values, err = tree.getRangeProof(key, nil, 2) + for i, key := range allkeys { + var keys, values [][]byte + proof, keys, values, err = tree.getRangeProof(key, nil, 2) require.Nil(t, err) require.Equal(t, append([]byte("value_for_"), key...), values[0], ) + require.Equal(t, key, keys[0]) require.Nil(t, proof.Verify(root)) require.Nil(t, proof.VerifyAbsence(cpIncr(key))) - if i < len(keys)-1 { - // There should be 2 items as per limit. - if len(values) != 2 { - printNode(tree.ndb, tree.root, 0) - } - require.Equal(t, 2, len(values), proof.String()) - if i < len(keys)-2 { + require.Equal(t, 1, len(keys), proof.String()) + require.Equal(t, 1, len(values), proof.String()) + if i < len(allkeys)-1 { + if i < len(allkeys)-2 { // No last item... not a proof of absence of large key. require.NotNil(t, proof.VerifyAbsence(bytes.Repeat([]byte{0xFF}, 20)), proof.String()) } else { @@ -99,8 +101,6 @@ func TestTreeKeyExistsProof(t *testing.T) { require.Nil(t, proof.VerifyAbsence(bytes.Repeat([]byte{0xFF}, 20))) } } else { - // There should be 1 item since no more can be queried. - require.Equal(t, 1, len(values), values) // last item of tree... valid proof of absence of large key. require.Nil(t, proof.VerifyAbsence(bytes.Repeat([]byte{0xFF}, 20))) } @@ -118,29 +118,41 @@ func TestTreeKeyInRangeProofs(t *testing.T) { } root := tree.Hash() + // For spacing: + T := 10 + nil______ := []byte(nil) + cases := []struct { - startKey byte - endKey byte - keys []byte // one byte per key. + start byte + end byte + pkeys []byte // proof keys, one byte per key. + vals []byte // keys and values, one byte per key. + lidx int64 // proof left index (index of first proof key). + pnc bool // does panic }{ - {startKey: 0x0a, endKey: 0xf7, keys: keys[:10]}, - {startKey: 0x0a, endKey: 0xf8, keys: keys[:10]}, - {startKey: 0x0, endKey: 0xff, keys: keys[:]}, - {startKey: 0x14, endKey: 0xe4, keys: keys[1:9]}, - {startKey: 0x14, endKey: 0xe5, keys: keys[1:9]}, - {startKey: 0x14, endKey: 0xe6, keys: keys[1:10]}, - {startKey: 0x14, endKey: 0xf1, keys: keys[1:10]}, - {startKey: 0x14, endKey: 0xf7, keys: keys[1:10]}, - {startKey: 0x14, endKey: 0xff, keys: keys[1:10]}, - {startKey: 0x2e, endKey: 0x31, keys: keys[2:4]}, - {startKey: 0x2e, endKey: 0x32, keys: keys[2:4]}, - {startKey: 0x2f, endKey: 0x32, keys: keys[2:4]}, - {startKey: 0x2e, endKey: 0x31, keys: keys[2:4]}, - {startKey: 0x2e, endKey: 0x2f, keys: keys[2:3]}, - {startKey: 0x12, endKey: 0x31, keys: keys[1:4]}, - {startKey: 0xf8, endKey: 0xff, keys: keys[9:10]}, - {startKey: 0x12, endKey: 0x20, keys: keys[1:3]}, - {startKey: 0x0, endKey: 0x09, keys: keys[0:1]}, + {start: 0x0a, end: 0xf7, pkeys: keys[0:T], vals: keys[0:9], lidx: 0}, // #0 + {start: 0x0a, end: 0xf8, pkeys: keys[0:T], vals: keys[0:T], lidx: 0}, // #1 + {start: 0x00, end: 0xff, pkeys: keys[0:T], vals: keys[0:T], lidx: 0}, // #2 + {start: 0x14, end: 0xe4, pkeys: keys[1:9], vals: keys[2:8], lidx: 1}, // #3 + {start: 0x14, end: 0xe5, pkeys: keys[1:9], vals: keys[2:9], lidx: 1}, // #4 + {start: 0x14, end: 0xe6, pkeys: keys[1:T], vals: keys[2:9], lidx: 1}, // #5 + {start: 0x14, end: 0xf1, pkeys: keys[1:T], vals: keys[2:9], lidx: 1}, // #6 + {start: 0x14, end: 0xf7, pkeys: keys[1:T], vals: keys[2:9], lidx: 1}, // #7 + {start: 0x14, end: 0xff, pkeys: keys[1:T], vals: keys[2:T], lidx: 1}, // #8 + {start: 0x2e, end: 0x31, pkeys: keys[2:4], vals: keys[2:3], lidx: 2}, // #9 + {start: 0x2e, end: 0x32, pkeys: keys[2:4], vals: keys[2:3], lidx: 2}, // #10 + {start: 0x2f, end: 0x32, pkeys: keys[2:4], vals: nil______, lidx: 2}, // #11 + {start: 0x2e, end: 0x31, pkeys: keys[2:4], vals: keys[2:3], lidx: 2}, // #12 + {start: 0x2e, end: 0x2f, pkeys: keys[2:3], vals: keys[2:3], lidx: 2}, // #13 + {start: 0x12, end: 0x31, pkeys: keys[1:4], vals: keys[2:3], lidx: 1}, // #14 + {start: 0xf8, end: 0xff, pkeys: keys[9:T], vals: nil______, lidx: 9}, // #15 + {start: 0x12, end: 0x20, pkeys: keys[1:3], vals: nil______, lidx: 1}, // #16 + {start: 0x00, end: 0x09, pkeys: keys[0:1], vals: nil______, lidx: 0}, // #17 + {start: 0xf7, end: 0x00, pnc: true}, // #18 + {start: 0xf8, end: 0x00, pnc: true}, // #19 + {start: 0x10, end: 0x10, pnc: true}, // #20 + {start: 0x12, end: 0x12, pnc: true}, // #21 + {start: 0xff, end: 0xf7, pnc: true}, // #22 } // fmt.Println("PRINT TREE") @@ -149,24 +161,36 @@ func TestTreeKeyInRangeProofs(t *testing.T) { for i, c := range cases { t.Logf("case %v", i) - startKey := []byte{c.startKey} - endKey := []byte{c.endKey} + start := []byte{c.start} + end := []byte{c.end} + + if c.pnc { + require.Panics(func() { tree.GetRangeWithProof(start, end, 0) }) + continue + } // Compute range proof. - keys, values, proof, err := tree.GetRangeWithProof(startKey, endKey, 0) + keys, values, proof, err := tree.GetRangeWithProof(start, end, 0) require.NoError(err, "%+v", err) - require.Equal(c.keys, flatten(keys)) - require.Equal(c.keys, flatten(values)) + require.Equal(c.pkeys, flatten(proof.Keys())) + require.Equal(c.vals, flatten(keys)) + require.Equal(c.vals, flatten(values)) + require.Equal(c.lidx, proof.LeftIndex()) // Verify that proof is valid. err = proof.Verify(root) - require.NoError(err, "%+v", err) verifyProof(t, proof, root) - // Verify each value. - for i, key := range c.keys { - err := proof.VerifyItem(i, []byte{key}, []byte{key}) + // Verify each value of pkeys. + for _, key := range c.pkeys { + err := proof.VerifyItem([]byte{key}, []byte{key}) + require.NoError(err) + } + + // Verify each value of vals. + for _, key := range c.vals { + err := proof.VerifyItem([]byte{key}, []byte{key}) require.NoError(err) } } @@ -203,10 +227,6 @@ func verifyProof(t *testing.T, proof *RangeProof, root []byte) { proofBytes, badProofBytes) } } - - // targetted changes fails... - proof.RootHash = test.MutateByteSlice(proof.RootHash) - assert.Error(t, proof.Verify(root)) } //---------------------------------------- diff --git a/tree_test.go b/tree_test.go index a2302285a..4308d6e78 100644 --- a/tree_test.go +++ b/tree_test.go @@ -987,7 +987,7 @@ func TestVersionedTreeProofs(t *testing.T) { require.NoError(err) require.EqualValues(val, []byte("v1")) require.NoError(proof.Verify(root1), proof.String()) - require.NoError(proof.VerifyItem(0, []byte("k2"), val)) + require.NoError(proof.VerifyItem([]byte("k2"), val)) val, proof, err = tree.GetVersionedWithProof([]byte("k4"), 1) require.NoError(err) @@ -999,13 +999,13 @@ func TestVersionedTreeProofs(t *testing.T) { require.NoError(err) require.EqualValues(val, []byte("v2")) require.NoError(proof.Verify(root2), proof.String()) - require.NoError(proof.VerifyItem(0, []byte("k2"), val)) + require.NoError(proof.VerifyItem([]byte("k2"), val)) val, proof, err = tree.GetVersionedWithProof([]byte("k1"), 2) require.NoError(err) require.EqualValues(val, []byte("v1")) require.NoError(proof.Verify(root2)) - require.NoError(proof.VerifyItem(0, []byte("k1"), val)) + require.NoError(proof.VerifyItem([]byte("k1"), val)) val, proof, err = tree.GetVersionedWithProof([]byte("k2"), 3) @@ -1036,7 +1036,7 @@ func TestVersionedTreeHash(t *testing.T) { require.NoError(err) require.EqualValues(val, []byte("F")) require.NoError(proof.Verify(hash2)) - require.NoError(proof.VerifyItem(0, []byte("I"), val)) + require.NoError(proof.VerifyItem([]byte("I"), val)) } func TestNilValueSemantics(t *testing.T) { diff --git a/version.go b/version.go index b9241d031..4c734cb35 100644 --- a/version.go +++ b/version.go @@ -1,3 +1,3 @@ package iavl -const Version = "0.8.1" +const Version = "0.9.1" From 23c15367f182a02b2834d79b6d4c65b0a019a91a Mon Sep 17 00:00:00 2001 From: Ismail Khoffi Date: Tue, 3 Jul 2018 18:34:44 +0100 Subject: [PATCH 02/10] Various lints (#80) * adhere to: ineffassign, misspell, golint * unexport helper function & godoc compatibility for package comments * _PathToLeaf -> pathToLeaf * remove obsolete proof-types from doc * comment exported method and unexport debug helper method * remove unused private methods * remove unnecessary else blocks (golint): * remove unused named returns * minor changes to make code more idiomatic * consistent receiver names * omit type from declaration * remove brackets from single return val Signed-off-by: Liamsi --- Gopkg.lock | 3 +- basic_test.go | 10 +-- doc.go | 8 ++- node.go | 162 ++++++++++++++++++++---------------------- proof.go | 41 ++++++----- proof_path.go | 10 +-- proof_range.go | 39 +++++----- proof_test.go | 4 +- testutils_test.go | 17 +---- tree.go | 17 ++--- tree_dotgraph_test.go | 2 +- tree_fuzz_test.go | 4 +- util.go | 1 + version.go | 2 +- versioned_tree.go | 1 + 15 files changed, 153 insertions(+), 168 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index a4b4f637e..9a4174900 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -102,6 +102,7 @@ [[projects]] name = "github.com/tendermint/tendermint" packages = [ + "crypto/tmhash", "libs/common", "libs/db", "libs/log", @@ -122,6 +123,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "ad12afa101079a7f5bcd70176fbbab42fd52e844b6c74c0fcf81315d33053f90" + inputs-digest = "06b259c645a1a884f870ef81ab220a7302b156f6fc1f1500779d540a7eed0d49" solver-name = "gps-cdcl" solver-version = 1 diff --git a/basic_test.go b/basic_test.go index 699177fde..5dca7cb20 100644 --- a/basic_test.go +++ b/basic_test.go @@ -12,7 +12,7 @@ import ( ) func TestBasic(t *testing.T) { - var tree *Tree = NewTree(nil, 0) + tree := NewTree(nil, 0) up := tree.Set([]byte("1"), []byte("one")) if up { t.Error("Did not expect an update (should have been create)") @@ -220,7 +220,7 @@ func TestIntegration(t *testing.T) { } records := make([]*record, 400) - var tree *Tree = NewTree(nil, 0) + tree := NewTree(nil, 0) randomRecord := func() *record { return &record{randstr(20), randstr(20)} @@ -302,7 +302,7 @@ func TestIterateRange(t *testing.T) { } sort.Strings(keys) - var tree *Tree = NewTree(nil, 0) + tree := NewTree(nil, 0) // insert all the data for _, r := range records { @@ -394,7 +394,7 @@ func TestProof(t *testing.T) { // Construct some random tree db := db.NewMemDB() - var tree *VersionedTree = NewVersionedTree(db, 100) + tree := NewVersionedTree(db, 100) for i := 0; i < 1000; i++ { key, value := randstr(20), randstr(20) tree.Set([]byte(key), []byte(value)) @@ -423,7 +423,7 @@ func TestProof(t *testing.T) { func TestTreeProof(t *testing.T) { db := db.NewMemDB() - var tree *Tree = NewTree(db, 100) + tree := NewTree(db, 100) // should get false for proof with nil root _, _, err := tree.GetWithProof([]byte("foo")) diff --git a/doc.go b/doc.go index e35179273..7e4891bcb 100644 --- a/doc.go +++ b/doc.go @@ -1,3 +1,7 @@ +// Package iavl implements a versioned, snapshottable (immutable) AVL+ tree +// for persisting key-value pairs. +// +// // Basic usage of VersionedTree. // // import "github.com/tendermint/iavl" @@ -23,12 +27,12 @@ // Proof of existence: // // root := tree.Hash() -// val, proof, err := tree.GetVersionedWithProof([]byte("bob"), 2) // "xyz", KeyProof, nil +// val, proof, err := tree.GetVersionedWithProof([]byte("bob"), 2) // "xyz", RangeProof, nil // proof.Verify([]byte("bob"), val, root) // nil // // Proof of absence: // -// _, proof, err = tree.GetVersionedWithProof([]byte("tom"), 2) // nil, KeyProof, nil +// _, proof, err = tree.GetVersionedWithProof([]byte("tom"), 2) // nil, RangeProof, nil // proof.Verify([]byte("tom"), nil, root) // nil // // Now we delete an old version: diff --git a/node.go b/node.go index fd86eff57..307412c33 100644 --- a/node.go +++ b/node.go @@ -41,46 +41,50 @@ func NewNode(key []byte, value []byte, version int64) *Node { // MakeNode constructs an *Node from an encoded byte slice. // -// The new node doesn't have its hash saved or set. The caller must set it +// The new node doesn't have its hash saved or set. The caller must set it // afterwards. -func MakeNode(buf []byte) (node *Node, err cmn.Error) { - node = &Node{} - n := 0 - cause := error(nil) +func MakeNode(buf []byte) (*Node, cmn.Error) { - // Read node header. - - node.height, n, cause = amino.DecodeInt8(buf) + // Read node header (height, size, version, key). + height, n, cause := amino.DecodeInt8(buf) if cause != nil { return nil, cmn.ErrorWrap(cause, "decoding node.height") } buf = buf[n:] - node.size, n, cause = amino.DecodeVarint(buf) + size, n, cause := amino.DecodeVarint(buf) if cause != nil { return nil, cmn.ErrorWrap(cause, "decoding node.size") } buf = buf[n:] - node.version, n, cause = amino.DecodeVarint(buf) + ver, n, cause := amino.DecodeVarint(buf) if cause != nil { return nil, cmn.ErrorWrap(cause, "decoding node.version") } buf = buf[n:] - node.key, n, cause = amino.DecodeByteSlice(buf) + key, n, cause := amino.DecodeByteSlice(buf) if cause != nil { return nil, cmn.ErrorWrap(cause, "decoding node.key") } buf = buf[n:] + node := &Node{ + height: height, + size: size, + version: ver, + key: key, + } + // Read node body. if node.isLeaf() { - node.value, _, cause = amino.DecodeByteSlice(buf) + val, _, cause := amino.DecodeByteSlice(buf) if cause != nil { return nil, cmn.ErrorWrap(cause, "decoding node.value") } + node.value = val } else { // Read children. leftHash, n, cause := amino.DecodeByteSlice(buf) if cause != nil { @@ -145,9 +149,8 @@ func (node *Node) has(t *Tree, key []byte) (has bool) { } if bytes.Compare(key, node.key) < 0 { return node.getLeftNode(t).has(t, key) - } else { - return node.getRightNode(t).has(t, key) } + return node.getRightNode(t).has(t, key) } // Get a key under the node. @@ -165,31 +168,28 @@ func (node *Node) get(t *Tree, key []byte) (index int64, value []byte) { if bytes.Compare(key, node.key) < 0 { return node.getLeftNode(t).get(t, key) - } else { - rightNode := node.getRightNode(t) - index, value = rightNode.get(t, key) - index += node.size - rightNode.size - return index, value } + rightNode := node.getRightNode(t) + index, value = rightNode.get(t, key) + index += node.size - rightNode.size + return index, value } func (node *Node) getByIndex(t *Tree, index int64) (key []byte, value []byte) { if node.isLeaf() { if index == 0 { return node.key, node.value - } else { - return nil, nil - } - } else { - // TODO: could improve this by storing the - // sizes as well as left/right hash. - leftNode := node.getLeftNode(t) - if index < leftNode.size { - return leftNode.getByIndex(t, index) - } else { - return node.getRightNode(t).getByIndex(t, index-leftNode.size) } + return nil, nil + } + // TODO: could improve this by storing the + // sizes as well as left/right hash. + leftNode := node.getLeftNode(t) + + if index < leftNode.size { + return leftNode.getByIndex(t, index) } + return node.getRightNode(t).getByIndex(t, index-leftNode.size) } // Computes the hash of the node without computing its descendants. Must be @@ -387,35 +387,33 @@ func (node *Node) set(t *Tree, key []byte, value []byte) ( if updated { return node, updated, orphaned - } else { - node.calcHeightAndSize(t) - newNode, balanceOrphaned := node.balance(t) - return newNode, updated, append(orphaned, balanceOrphaned...) } + node.calcHeightAndSize(t) + newNode, balanceOrphaned := node.balance(t) + return newNode, updated, append(orphaned, balanceOrphaned...) } } -// newHash/newNode: The new hash or node to replace node after remove. -// newKey: new leftmost leaf key for tree after successfully removing 'key' if changed. -// value: removed value. -func (node *Node) remove(t *Tree, key []byte) ( - newHash []byte, newNode *Node, newKey []byte, value []byte, orphaned []*Node, -) { +// removes the node corresponding to the passed key and balances the tree. +// It returns: +// - the hash of the new node (or nil if the node is the one removed) +// - the node that replaces the orig. node after remove +// - new leftmost leaf key for tree after successfully removing 'key' if changed. +// - the removed value +// - the orphaned nodes. +func (node *Node) remove(t *Tree, key []byte) ([]byte, *Node, []byte, []byte, []*Node) { version := t.version + 1 if node.isLeaf() { if bytes.Equal(key, node.key) { return nil, nil, nil, node.value, []*Node{node} } - return node.hash, node, nil, nil, orphaned + return node.hash, node, nil, nil, nil } + // node.key < key; we go to the left to find the key: if bytes.Compare(key, node.key) < 0 { - var newLeftHash []byte - var newLeftNode *Node - - newLeftHash, newLeftNode, newKey, value, orphaned = - node.getLeftNode(t).remove(t, key) + newLeftHash, newLeftNode, newKey, value, orphaned := node.getLeftNode(t).remove(t, key) if len(orphaned) == 0 { return node.hash, node, nil, value, orphaned @@ -430,30 +428,26 @@ func (node *Node) remove(t *Tree, key []byte) ( newNode, balanceOrphaned := newNode.balance(t) return newNode.hash, newNode, newKey, value, append(orphaned, balanceOrphaned...) - } else { - var newRightHash []byte - var newRightNode *Node - - newRightHash, newRightNode, newKey, value, orphaned = - node.getRightNode(t).remove(t, key) - - if len(orphaned) == 0 { - return node.hash, node, nil, value, orphaned - } else if newRightHash == nil && newRightNode == nil { // right node held value, was removed - return node.leftHash, node.leftNode, nil, value, orphaned - } - orphaned = append(orphaned, node) + } + // node.key >= key; either found or look to the right: + newRightHash, newRightNode, newKey, value, orphaned := node.getRightNode(t).remove(t, key) - newNode := node.clone(version) - newNode.rightHash, newNode.rightNode = newRightHash, newRightNode - if newKey != nil { - newNode.key = newKey - } - newNode.calcHeightAndSize(t) - newNode, balanceOrphaned := newNode.balance(t) + if len(orphaned) == 0 { + return node.hash, node, nil, value, orphaned + } else if newRightHash == nil && newRightNode == nil { // right node held value, was removed + return node.leftHash, node.leftNode, nil, value, orphaned + } + orphaned = append(orphaned, node) - return newNode.hash, newNode, nil, value, append(orphaned, balanceOrphaned...) + newNode := node.clone(version) + newNode.rightHash, newNode.rightNode = newRightHash, newRightNode + if newKey != nil { + newNode.key = newKey } + newNode.calcHeightAndSize(t) + newNode, balanceOrphaned := newNode.balance(t) + + return newNode.hash, newNode, nil, value, append(orphaned, balanceOrphaned...) } func (node *Node) getLeftNode(t *Tree) *Node { @@ -531,34 +525,32 @@ func (node *Node) balance(t *Tree) (newSelf *Node, orphaned []*Node) { // Left Left Case newNode, orphaned := node.rotateRight(t) return newNode, []*Node{orphaned} - } else { - // Left Right Case - var leftOrphaned *Node + } + // Left Right Case + var leftOrphaned *Node - left := node.getLeftNode(t) - node.leftHash = nil - node.leftNode, leftOrphaned = left.rotateLeft(t) - newNode, rightOrphaned := node.rotateRight(t) + left := node.getLeftNode(t) + node.leftHash = nil + node.leftNode, leftOrphaned = left.rotateLeft(t) + newNode, rightOrphaned := node.rotateRight(t) - return newNode, []*Node{left, leftOrphaned, rightOrphaned} - } + return newNode, []*Node{left, leftOrphaned, rightOrphaned} } if balance < -1 { if node.getRightNode(t).calcBalance(t) <= 0 { // Right Right Case newNode, orphaned := node.rotateLeft(t) return newNode, []*Node{orphaned} - } else { - // Right Left Case - var rightOrphaned *Node + } + // Right Left Case + var rightOrphaned *Node - right := node.getRightNode(t) - node.rightHash = nil - node.rightNode, rightOrphaned = right.rotateRight(t) - newNode, leftOrphaned := node.rotateLeft(t) + right := node.getRightNode(t) + node.rightHash = nil + node.rightNode, rightOrphaned = right.rotateRight(t) + newNode, leftOrphaned := node.rotateLeft(t) - return newNode, []*Node{right, leftOrphaned, rightOrphaned} - } + return newNode, []*Node{right, leftOrphaned, rightOrphaned} } // Nothing changed return node, []*Node{} diff --git a/proof.go b/proof.go index 7a600ce1a..a87877048 100644 --- a/proof.go +++ b/proof.go @@ -34,10 +34,10 @@ type proofInnerNode struct { } func (pin proofInnerNode) String() string { - return pin.StringIndented("") + return pin.stringIndented("") } -func (pin proofInnerNode) StringIndented(indent string) string { +func (pin proofInnerNode) stringIndented(indent string) string { return fmt.Sprintf(`proofInnerNode{ %s Height: %v %s Size: %v @@ -97,10 +97,10 @@ type proofLeafNode struct { } func (pln proofLeafNode) String() string { - return pln.StringIndented("") + return pln.stringIndented("") } -func (pln proofLeafNode) StringIndented(indent string) string { +func (pln proofLeafNode) stringIndented(indent string) string { return fmt.Sprintf(`proofLeafNode{ %s Key: %v %s ValueHash: %X @@ -144,10 +144,14 @@ func (pln proofLeafNode) Hash() []byte { // a path to the least item. func (node *Node) PathToLeaf(t *Tree, key []byte) (PathToLeaf, *Node, error) { path := new(PathToLeaf) - val, err := node._PathToLeaf(t, key, path) + val, err := node.pathToLeaf(t, key, path) return *path, val, err } -func (node *Node) _PathToLeaf(t *Tree, key []byte, path *PathToLeaf) (*Node, error) { + +// pathToLeaf is a helper which recursively constructs the PathToLeaf. +// As an optimization the already constructed path is passed in as an argument +// and is shared among recursive calls. +func (node *Node) pathToLeaf(t *Tree, key []byte, path *PathToLeaf) (*Node, error) { if node.height == 0 { if bytes.Equal(node.key, key) { return node, nil @@ -165,19 +169,18 @@ func (node *Node) _PathToLeaf(t *Tree, key []byte, path *PathToLeaf) (*Node, err Right: node.getRightNode(t).hash, } *path = append(*path, pin) - n, err := node.getLeftNode(t)._PathToLeaf(t, key, path) - return n, err - } else { - // right side - pin := proofInnerNode{ - Height: node.height, - Size: node.size, - Version: node.version, - Left: node.getLeftNode(t).hash, - Right: nil, - } - *path = append(*path, pin) - n, err := node.getRightNode(t)._PathToLeaf(t, key, path) + n, err := node.getLeftNode(t).pathToLeaf(t, key, path) return n, err } + // right side + pin := proofInnerNode{ + Height: node.height, + Size: node.size, + Version: node.version, + Left: node.getLeftNode(t).hash, + Right: nil, + } + *path = append(*path, pin) + n, err := node.getRightNode(t).pathToLeaf(t, key, path) + return n, err } diff --git a/proof_path.go b/proof_path.go index e865a3765..de366f338 100644 --- a/proof_path.go +++ b/proof_path.go @@ -23,8 +23,8 @@ func (pwl pathWithLeaf) StringIndented(indent string) string { %s Path: %v %s Leaf: %v %s}`, - indent, pwl.Path.StringIndented(indent+" "), - indent, pwl.Leaf.StringIndented(indent+" "), + indent, pwl.Path.stringIndented(indent+" "), + indent, pwl.Leaf.stringIndented(indent+" "), indent) } @@ -51,10 +51,10 @@ func (pwl pathWithLeaf) computeRootHash() []byte { type PathToLeaf []proofInnerNode func (pl PathToLeaf) String() string { - return pl.StringIndented("") + return pl.stringIndented("") } -func (pl PathToLeaf) StringIndented(indent string) string { +func (pl PathToLeaf) stringIndented(indent string) string { if len(pl) == 0 { return "empty-PathToLeaf" } @@ -64,7 +64,7 @@ func (pl PathToLeaf) StringIndented(indent string) string { strs[i] = fmt.Sprintf("... (%v total)", len(pl)) break } - strs[i] = fmt.Sprintf("%v:%v", i, pin.StringIndented(indent+" ")) + strs[i] = fmt.Sprintf("%v:%v", i, pin.stringIndented(indent+" ")) } return fmt.Sprintf(`PathToLeaf{ %s %v diff --git a/proof_range.go b/proof_range.go index ba9834084..cc12618f9 100644 --- a/proof_range.go +++ b/proof_range.go @@ -52,11 +52,11 @@ func (proof *RangeProof) String() string { func (proof *RangeProof) StringIndented(indent string) string { istrs := make([]string, 0, len(proof.InnerNodes)) for _, ptl := range proof.InnerNodes { - istrs = append(istrs, ptl.StringIndented(indent+" ")) + istrs = append(istrs, ptl.stringIndented(indent+" ")) } lstrs := make([]string, 0, len(proof.Leaves)) for _, leaf := range proof.Leaves { - lstrs = append(lstrs, leaf.StringIndented(indent+" ")) + lstrs = append(lstrs, leaf.stringIndented(indent+" ")) } return fmt.Sprintf(`RangeProof{ %s LeftPath: %v @@ -68,7 +68,7 @@ func (proof *RangeProof) StringIndented(indent string) string { %s (rootHash): %X %s (treeEnd): %v %s}`, - indent, proof.LeftPath.StringIndented(indent+" "), + indent, proof.LeftPath.stringIndented(indent+" "), indent, indent, strings.Join(istrs, "\n"+indent+" "), indent, @@ -126,9 +126,8 @@ func (proof *RangeProof) VerifyAbsence(key []byte) error { if cmp < 0 { if proof.LeftPath.isLeftmost() { return nil - } else { - return cmn.NewError("absence not proved by left path") } + return cmn.NewError("absence not proved by left path") } else if cmp == 0 { return cmn.NewError("absence disproved via first item #0") } @@ -150,7 +149,7 @@ func (proof *RangeProof) VerifyAbsence(key []byte) error { } else { if i == len(proof.Leaves)-1 { // If last item, check whether - // it's the last item in teh tree. + // it's the last item in the tree. } continue @@ -165,9 +164,8 @@ func (proof *RangeProof) VerifyAbsence(key []byte) error { // It's not a valid absence proof. if len(proof.Leaves) < 2 { return cmn.NewError("absence not proved by right leaf (need another leaf?)") - } else { - return cmn.NewError("absence not proved by right leaf") } + return cmn.NewError("absence not proved by right leaf") } // Verify that proof is valid. @@ -179,7 +177,7 @@ func (proof *RangeProof) Verify(root []byte) error { return err } -func (proof *RangeProof) verify(root []byte) (err error) { +func (proof *RangeProof) verify(root []byte) error { rootHash := proof.rootHash if rootHash == nil { derivedHash, err := proof.computeRootHash() @@ -190,9 +188,8 @@ func (proof *RangeProof) verify(root []byte) (err error) { } if !bytes.Equal(rootHash, root) { return cmn.ErrorWrap(ErrInvalidRoot, "root hash doesn't match") - } else { - proof.rootVerified = true } + proof.rootVerified = true return nil } @@ -305,12 +302,13 @@ func (proof *RangeProof) _computeRootHash() (rootHash []byte, treeEnd bool, err /////////////////////////////////////////////////////////////////////////////// // keyStart is inclusive and keyEnd is exclusive. +// Returns the range-proof and the included keys and values. // If keyStart or keyEnd don't exist, the leaf before keyStart // or after keyEnd will also be included, but not be included in values. // If keyEnd-1 exists, no later leaves will be included. // If keyStart >= keyEnd and both not nil, panics. // Limit is never exceeded. -func (t *Tree) getRangeProof(keyStart, keyEnd []byte, limit int) (proof *RangeProof, keys, values [][]byte, err error) { +func (t *Tree) getRangeProof(keyStart, keyEnd []byte, limit int) (*RangeProof, [][]byte, [][]byte, error) { if keyStart != nil && keyEnd != nil && bytes.Compare(keyStart, keyEnd) >= 0 { panic("if keyStart and keyEnd are present, need keyStart < keyEnd.") } @@ -327,17 +325,18 @@ func (t *Tree) getRangeProof(keyStart, keyEnd []byte, limit int) (proof *RangePr if err != nil { // Key doesn't exist, but instead we got the prev leaf (or the // first or last leaf), which provides proof of absence). - err = nil + // err = nil isn't necessary as we do not use it in the returns below } startOK := keyStart == nil || bytes.Compare(keyStart, left.key) <= 0 endOK := keyEnd == nil || bytes.Compare(left.key, keyEnd) < 0 // If left.key is in range, add it to key/values. + var keys, values [][]byte if startOK && endOK { keys = append(keys, left.key) // == keyStart values = append(values, left.value) } // Either way, add to proof leaves. - var leaves = []proofLeafNode{proofLeafNode{ + var leaves = []proofLeafNode{{ Key: left.key, ValueHash: tmhash.Sum(left.value), Version: left.version, @@ -387,9 +386,9 @@ func (t *Tree) getRangeProof(keyStart, keyEnd []byte, limit int) (proof *RangePr pn.Right != nil && !bytes.Equal(pn.Right, node.rightHash) { // We've diverged, so start appending to inners. - pathCount = -1 + pathCount-- } else { - pathCount += 1 + pathCount++ } } } @@ -405,7 +404,7 @@ func (t *Tree) getRangeProof(keyStart, keyEnd []byte, limit int) (proof *RangePr ValueHash: tmhash.Sum(node.value), Version: node.version, }) - leafCount += 1 + leafCount++ // Maybe terminate because we found enough leaves. if limit > 0 && limit <= leafCount { return true @@ -458,12 +457,10 @@ func (t *Tree) GetWithProof(key []byte) (value []byte, proof *RangeProof, err er if len(values) > 0 { if !bytes.Equal(proof.Leaves[0].Key, key) { return nil, proof, nil - } else { - return values[0], proof, nil } - } else { - return nil, proof, nil + return values[0], proof, nil } + return nil, proof, nil } return nil, nil, cmn.ErrorWrap(err, "could not construct any proof") } diff --git a/proof_test.go b/proof_test.go index 4410ac6aa..53cc5c977 100644 --- a/proof_test.go +++ b/proof_test.go @@ -120,6 +120,8 @@ func TestTreeKeyInRangeProofs(t *testing.T) { // For spacing: T := 10 + // disable: don't use underscores in Go names; var nil______ should be nil (golint) + // nolint nil______ := []byte(nil) cases := []struct { @@ -222,7 +224,7 @@ func verifyProof(t *testing.T, proof *RangeProof, root []byte) { } // may be invalid... errors are okay if err == nil { - assert.Error(t, badProof.Verify(root), + assert.Errorf(t, badProof.Verify(root), "Proof was still valid after a random mutation:\n%X\n%X", proofBytes, badProofBytes) } diff --git a/testutils_test.go b/testutils_test.go index d39dbe23d..1e727fb52 100644 --- a/testutils_test.go +++ b/testutils_test.go @@ -13,18 +13,6 @@ import ( "github.com/tendermint/tendermint/libs/db" ) -func dummyPathToLeaf(t *Tree, key []byte) PathToLeaf { - path, _, err := t.root.PathToLeaf(t, key) - if err != nil { - panic(err) - } - return path -} - -func dummyLeafNode(key, val []byte) proofLeafNode { - return proofLeafNode{key, val, 1} -} - func randstr(length int) string { return cmn.RandStr(length) } @@ -78,9 +66,8 @@ func T(n *Node) *Tree { func P(n *Node) string { if n.height == 0 { return fmt.Sprintf("%v", b2i(n.key)) - } else { - return fmt.Sprintf("(%v %v)", P(n.leftNode), P(n.rightNode)) } + return fmt.Sprintf("(%v %v)", P(n.leftNode), P(n.rightNode)) } func randBytes(length int) []byte { @@ -101,7 +88,7 @@ func (t *traverser) view(key, value []byte) bool { t.first = string(key) } t.last = string(key) - t.count += 1 + t.count++ return false } diff --git a/tree.go b/tree.go index 676618b43..c92901532 100644 --- a/tree.go +++ b/tree.go @@ -175,9 +175,8 @@ func (t *Tree) Iterate(fn func(key []byte, value []byte) bool) (stopped bool) { return t.root.traverse(t, true, func(node *Node) bool { if node.height == 0 { return fn(node.key, node.value) - } else { - return false } + return false }) } @@ -190,9 +189,8 @@ func (t *Tree) IterateRange(start, end []byte, ascending bool, fn func(key []byt return t.root.traverseInRange(t, start, end, ascending, false, 0, func(node *Node, _ uint8) bool { if node.height == 0 { return fn(node.key, node.value) - } else { - return false } + return false }) } @@ -205,19 +203,18 @@ func (t *Tree) IterateRangeInclusive(start, end []byte, ascending bool, fn func( return t.root.traverseInRange(t, start, end, ascending, true, 0, func(node *Node, _ uint8) bool { if node.height == 0 { return fn(node.key, node.value, node.version) - } else { - return false } + return false }) } // Clone creates a clone of the tree. // Used internally by VersionedTree. -func (tree *Tree) clone() *Tree { +func (t *Tree) clone() *Tree { return &Tree{ - root: tree.root, - ndb: tree.ndb, - version: tree.version, + root: t.root, + ndb: t.ndb, + version: t.version, } } diff --git a/tree_dotgraph_test.go b/tree_dotgraph_test.go index a2c80a3e9..02b245c5d 100644 --- a/tree_dotgraph_test.go +++ b/tree_dotgraph_test.go @@ -6,7 +6,7 @@ import ( ) func TestWriteDOTGraph(t *testing.T) { - var tree *Tree = NewTree(nil, 0) + tree := NewTree(nil, 0) for _, ikey := range []byte{ 0x0a, 0x11, 0x2e, 0x32, 0x50, 0x72, 0x99, 0xa1, 0xe4, 0xf7, } { diff --git a/tree_fuzz_test.go b/tree_fuzz_test.go index 5825f64d1..59d693153 100644 --- a/tree_fuzz_test.go +++ b/tree_fuzz_test.go @@ -45,8 +45,8 @@ func (p *program) addInstruction(i instruction) { p.instructions = append(p.instructions, i) } -func (prog *program) size() int { - return len(prog.instructions) +func (p *program) size() int { + return len(p.instructions) } type instruction struct { diff --git a/util.go b/util.go index fefb558f8..96f754189 100644 --- a/util.go +++ b/util.go @@ -6,6 +6,7 @@ import ( "sort" ) +// PrintTree prints the whole tree in an indented form. func PrintTree(tree *Tree) { ndb, root := tree.ndb, tree.root printNode(ndb, root, 0) diff --git a/version.go b/version.go index 4417a9da3..526fcc68a 100644 --- a/version.go +++ b/version.go @@ -1,4 +1,4 @@ package iavl - +// Version of iavl. const Version = "0.9.1" diff --git a/versioned_tree.go b/versioned_tree.go index 3e399d11e..7d8108ea8 100644 --- a/versioned_tree.go +++ b/versioned_tree.go @@ -8,6 +8,7 @@ import ( dbm "github.com/tendermint/tendermint/libs/db" ) +// ErrVersionDoesNotExist is returned if a requested version does not exist. var ErrVersionDoesNotExist = fmt.Errorf("version does not exist") // VersionedTree is a persistent tree which keeps track of versions. From 9c3271a1047aef4248b46dcc6997e4bb9b9b6184 Mon Sep 17 00:00:00 2001 From: Ismail Khoffi Date: Tue, 3 Jul 2018 22:47:11 +0100 Subject: [PATCH 03/10] release v0.9.2 (#82) - bump version - update changelog Signed-off-by: Liamsi --- CHANGELOG.md | 6 ++++++ version.go | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0aa35a4cb..4ace77964 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 0.9.2 (July 3, 2018) + +IMPROVEMTS + +- some minor changes: mainly lints, updated parts of documentation, unexported some helpers (#80) + ## 0.9.1 (July 1, 2018) IMPROVEMENTS diff --git a/version.go b/version.go index 526fcc68a..9efd11145 100644 --- a/version.go +++ b/version.go @@ -1,4 +1,4 @@ package iavl // Version of iavl. -const Version = "0.9.1" +const Version = "0.9.2" From 4097d89e4d81be4656c3daff228eced807f626b0 Mon Sep 17 00:00:00 2001 From: Jeremiah Andrews Date: Wed, 25 Jul 2018 07:40:09 -0700 Subject: [PATCH 04/10] Fix benchmark scripts for current code (#89) * Fix benchmark scripts for current code * Add benchmark result --- benchmarks/README.md | 16 ++---- .../digial-ocean-32gb-18.04-a3d16d8.txt | 57 +++++++++++++++++++ benchmarks/setup/INSTALL_ROOT.sh | 8 +-- benchmarks/setup/INSTALL_USER.sh | 26 --------- benchmarks/setup/RUN_BENCHMARKS.sh | 26 +++++++-- 5 files changed, 84 insertions(+), 49 deletions(-) create mode 100644 benchmarks/results/digial-ocean-32gb-18.04-a3d16d8.txt delete mode 100755 benchmarks/setup/INSTALL_USER.sh diff --git a/benchmarks/README.md b/benchmarks/README.md index be253d827..a6295385c 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -2,7 +2,7 @@ These instructions are mainly for running the benchmarks on an cloud instance that is intended to be thrown away, not on a dev machine. Be careful with the install scripts locally. -This has only been tested on Ubuntu 16.04. It *should* work on Ubuntu 14.04 as well. It *may* work on Debian, but has never been tested. +This has only been tested on Ubuntu 16.04 and 18.04. It *should* work on Ubuntu 14.04 as well. It *may* work on Debian, but has never been tested. ## Setting up the machine @@ -14,34 +14,26 @@ scp -r setup user@host: ssh user@host ``` -Run the install scripts (once per machine) +Run the install script (once per machine) ``` cd setup chmod +x * sudo ./INSTALL_ROOT.sh -./INSTALL_USER.sh ``` ## Running the tests -Make sure the hostname is set to a good value for recording: - -``` -hostname -s -sudo hostname -``` - Run the benchmarks in a screen: ``` screen -~/RUN_BENCHMARKS.sh +./RUN_BENCHMARKS.sh ``` Copy them back from your local machine: ``` -scp user@host:go/src/github.com/tendermint/merkleeyes/iavl/benchmarks/results/* results +scp user@host:go/src/github.com/tendermint/iavl/results.txt results.txt git add results ``` diff --git a/benchmarks/results/digial-ocean-32gb-18.04-a3d16d8.txt b/benchmarks/results/digial-ocean-32gb-18.04-a3d16d8.txt new file mode 100644 index 000000000..095a2540d --- /dev/null +++ b/benchmarks/results/digial-ocean-32gb-18.04-a3d16d8.txt @@ -0,0 +1,57 @@ +cd benchmarks && \ + go test -bench=RandomBytes . && \ + go test -bench=Small . && \ + go test -bench=Medium . && \ + go test -bench=BenchmarkMemKeySizes . +goos: linux +goarch: amd64 +pkg: github.com/tendermint/iavl/benchmarks +BenchmarkRandomBytes/random-4-8 10000000 125 ns/op +BenchmarkRandomBytes/random-16-8 10000000 199 ns/op +BenchmarkRandomBytes/random-32-8 5000000 273 ns/op +BenchmarkRandomBytes/random-100-8 2000000 619 ns/op +BenchmarkRandomBytes/random-1000-8 300000 4655 ns/op +PASS +ok github.com/tendermint/iavl/benchmarks 8.571s +Init Tree took 0.90 MB +goos: linux +goarch: amd64 +pkg: github.com/tendermint/iavl/benchmarks +BenchmarkSmall/memdb-1000-100-4-10/query-miss-8 200000 7393 ns/op +BenchmarkSmall/memdb-1000-100-4-10/query-hits-8 200000 9513 ns/op +BenchmarkSmall/memdb-1000-100-4-10/update-8 10000 274623 ns/op +BenchmarkSmall/memdb-1000-100-4-10/block-8 50 39110192 ns/op +Init Tree took 0.49 MB +BenchmarkSmall/goleveldb-1000-100-4-10/query-miss-8 200000 10313 ns/op +BenchmarkSmall/goleveldb-1000-100-4-10/query-hits-8 100000 14936 ns/op +BenchmarkSmall/goleveldb-1000-100-4-10/update-8 10000 173268 ns/op +BenchmarkSmall/goleveldb-1000-100-4-10/block-8 100 30316707 ns/op +Init Tree took 0.49 MB +BenchmarkSmall/leveldb-1000-100-4-10/query-miss-8 200000 9815 ns/op +BenchmarkSmall/leveldb-1000-100-4-10/query-hits-8 100000 13164 ns/op +BenchmarkSmall/leveldb-1000-100-4-10/update-8 10000 167816 ns/op +BenchmarkSmall/leveldb-1000-100-4-10/block-8 100 27400294 ns/op +PASS +ok github.com/tendermint/iavl/benchmarks 25.039s +Init Tree took 85.10 MB +goos: linux +goarch: amd64 +pkg: github.com/tendermint/iavl/benchmarks +BenchmarkMedium/memdb-100000-100-16-40/query-miss-8 100000 16066 ns/op +BenchmarkMedium/memdb-100000-100-16-40/query-hits-8 100000 16443 ns/op +BenchmarkMedium/memdb-100000-100-16-40/update-8 2000 1349641 ns/op +BenchmarkMedium/memdb-100000-100-16-40/block-8 5 275106058 ns/op +Init Tree took 47.22 MB +BenchmarkMedium/goleveldb-100000-100-16-40/query-miss-8 30000 38140 ns/op +BenchmarkMedium/goleveldb-100000-100-16-40/query-hits-8 30000 42500 ns/op +BenchmarkMedium/goleveldb-100000-100-16-40/update-8 10000 444647 ns/op +BenchmarkMedium/goleveldb-100000-100-16-40/block-8 30 57406398 ns/op +Init Tree took 43.70 MB +BenchmarkMedium/leveldb-100000-100-16-40/query-miss-8 30000 38214 ns/op +BenchmarkMedium/leveldb-100000-100-16-40/query-hits-8 30000 40582 ns/op +BenchmarkMedium/leveldb-100000-100-16-40/update-8 10000 498158 ns/op +BenchmarkMedium/leveldb-100000-100-16-40/block-8 30 59098590 ns/op +PASS +ok github.com/tendermint/iavl/benchmarks 49.686s +PASS +ok github.com/tendermint/iavl/benchmarks 0.008s diff --git a/benchmarks/setup/INSTALL_ROOT.sh b/benchmarks/setup/INSTALL_ROOT.sh index 632a90272..14b37b03d 100755 --- a/benchmarks/setup/INSTALL_ROOT.sh +++ b/benchmarks/setup/INSTALL_ROOT.sh @@ -1,12 +1,10 @@ -#!/bin/bash +#!/bin/sh apt-get update apt-get -y upgrade apt-get -y install make screen -GOFILE=go1.7.linux-amd64.tar.gz +GOFILE=go1.10.linux-amd64.tar.gz wget https://storage.googleapis.com/golang/${GOFILE} -tar xzf ${GOFILE} -mv go /usr/local/go1.7 -ln -s /usr/local/go1.7 /usr/local/go +tar -C /usr/local -xzf ${GOFILE} diff --git a/benchmarks/setup/INSTALL_USER.sh b/benchmarks/setup/INSTALL_USER.sh deleted file mode 100755 index 4695a610a..000000000 --- a/benchmarks/setup/INSTALL_USER.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -# This runs benchmarks, by default from develop branch of -# github.com/tendermint/merkleeyes/iavl -# You can customize this by optional command line args -# -# INSTALL_USER.sh [branch] [repouser] -# -# set repouser as your username to time your fork - -BRANCH=${1:-develop} -REPOUSER=${2:-tendermint} - -cat <<'EOF' > ~/.goenv -export GOROOT=/usr/local/go -export GOPATH=$HOME/go -export PATH=$GOPATH/bin:$GOROOT/bin:$PATH -EOF - -. ~/.goenv - -mkdir -p $GOPATH/src/github.com/tendermint -MERKLE=$GOPATH/src/github.com/tendermint/merkleeyes/iavl -git clone https://github.com/${REPOUSER}/merkleeyes/iavl.git $MERKLE -cd $MERKLE -git checkout ${BRANCH} diff --git a/benchmarks/setup/RUN_BENCHMARKS.sh b/benchmarks/setup/RUN_BENCHMARKS.sh index 6753f8e35..2bed5c1e2 100755 --- a/benchmarks/setup/RUN_BENCHMARKS.sh +++ b/benchmarks/setup/RUN_BENCHMARKS.sh @@ -1,10 +1,24 @@ #!/bin/sh -. ~/.goenv +# This runs benchmarks, by default from develop branch of +# github.com/tendermint/iavl +# You can customize this by optional command line args +# +# INSTALL_USER.sh [branch] [repouser] +# +# set repouser as your username to time your fork -MERKLE=$GOPATH/src/github.com/tendermint/merkleeyes/iavl -cd $MERKLE -git pull +BRANCH=${1:-develop} +REPOUSER=${2:-tendermint} + +export PATH=$PATH:/usr/local/go/bin +export PATH=$PATH:$HOME/go/bin +export GOPATH=$HOME/go + +go get -u github.com/${REPOUSER}/iavl +cd ~/go/src/github.com/${REPOUSER}/iavl +git checkout ${BRANCH} + +make get_vendor_deps +make bench > results.txt -make get_deps -make record From 85a9f23faf5b35d756cf38ba4d7a836aed0456e6 Mon Sep 17 00:00:00 2001 From: Dev Ojha Date: Thu, 26 Jul 2018 07:53:16 -0700 Subject: [PATCH 05/10] dep: Change tendermint dep to be ^v0.22.0 (#91) This is done to remove the need for an override constraint for tendermint in the sdk. Closes #90 --- CHANGELOG.md | 8 +++++- Gopkg.lock | 72 ++++++++++++++++++++++++++++++++++++++++++++-------- Gopkg.toml | 2 +- 3 files changed, 70 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ace77964..40e34dd72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,14 @@ # Changelog +## PENDING + +IMPROVEMENTS + +- Change tendermint dep to ^v0.22.0 + ## 0.9.2 (July 3, 2018) -IMPROVEMTS +IMPROVEMENTS - some minor changes: mainly lints, updated parts of documentation, unexported some helpers (#80) diff --git a/Gopkg.lock b/Gopkg.lock index 9a4174900..4ebc929aa 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -2,80 +2,115 @@ [[projects]] + digest = "1:a2c1d0e43bd3baaa071d1b9ed72c27d78169b2b269f71c105ac4ba34b1be4a39" name = "github.com/davecgh/go-spew" packages = ["spew"] + pruneopts = "UT" revision = "346938d642f2ec3594ed81d874461961cd0faa76" version = "v1.1.0" [[projects]] + digest = "1:d0309acc14b09c713e2fd1283be859c8e03f5f40c69e221410a0f652742b139c" name = "github.com/go-kit/kit" packages = [ "log", "log/level", - "log/term" + "log/term", ] + pruneopts = "UT" revision = "4dc7be5d2d12881735283bcab7352178e190fc71" version = "v0.6.0" [[projects]] + digest = "1:31a18dae27a29aa074515e43a443abfd2ba6deb6d69309d8d7ce789c45f34659" name = "github.com/go-logfmt/logfmt" packages = ["."] + pruneopts = "UT" revision = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5" version = "v0.3.0" [[projects]] + digest = "1:c4a2528ccbcabf90f9f3c464a5fc9e302d592861bbfd0b7135a7de8a943d0406" name = "github.com/go-stack/stack" packages = ["."] + pruneopts = "UT" revision = "259ab82a6cad3992b4e21ff5cac294ccb06474bc" version = "v1.7.0" [[projects]] + digest = "1:0cff52d4289e3d31494fb4e63d00c93be0f9d07ad1e3447d1dc300add4e32224" + name = "github.com/gogo/protobuf" + packages = [ + "gogoproto", + "proto", + "protoc-gen-gogo/descriptor", + ] + pruneopts = "UT" + revision = "636bf0302bc95575d69441b25a2603156ffdddf1" + version = "v1.1.1" + +[[projects]] + digest = "1:15042ad3498153684d09f393bbaec6b216c8eec6d61f63dff711de7d64ed8861" name = "github.com/golang/protobuf" packages = ["proto"] - revision = "925541529c1fa6821df4e44ce2723319eb2be768" - version = "v1.0.0" + pruneopts = "UT" + revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265" + version = "v1.1.0" [[projects]] branch = "master" + digest = "1:4a0c6bb4805508a6287675fac876be2ac1182539ca8a32468d8128882e9d5009" name = "github.com/golang/snappy" packages = ["."] + pruneopts = "UT" revision = "2e65f85255dbc3072edf28d6b5b8efc472979f5a" [[projects]] branch = "master" + digest = "1:39b27d1381a30421f9813967a5866fba35dc1d4df43a6eefe3b7a5444cb07214" name = "github.com/jmhodges/levigo" packages = ["."] + pruneopts = "UT" revision = "c42d9e0ca023e2198120196f842701bb4c55d7b9" [[projects]] branch = "master" + digest = "1:a64e323dc06b73892e5bb5d040ced475c4645d456038333883f58934abbf6f72" name = "github.com/kr/logfmt" packages = ["."] + pruneopts = "UT" revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0" [[projects]] + digest = "1:40e195917a951a8bf867cd05de2a46aaf1806c50cf92eebf4c16f78cd196f747" name = "github.com/pkg/errors" packages = ["."] + pruneopts = "UT" revision = "645ef00459ed84a119197bfb8d8205042c6df63d" version = "v0.8.0" [[projects]] + digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe" name = "github.com/pmezard/go-difflib" packages = ["difflib"] + pruneopts = "UT" revision = "792786c7400a136282c1664665ae0a8db921c6c2" version = "v1.0.0" [[projects]] + digest = "1:befd7181c2f92c9f05abf17cd7e15538919f19cf7871d794cacc45a4fb99c135" name = "github.com/stretchr/testify" packages = [ "assert", - "require" + "require", ] + pruneopts = "UT" revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686" version = "v1.2.2" [[projects]] branch = "master" + digest = "1:bd62f27525a36697564991b8e6071ff56afa99d3235261924a0212db5ce780bd" name = "github.com/syndtr/goleveldb" packages = [ "leveldb", @@ -89,40 +124,57 @@ "leveldb/opt", "leveldb/storage", "leveldb/table", - "leveldb/util" + "leveldb/util", ] + pruneopts = "UT" revision = "0d5a0ceb10cf9ab89fdd744cc8c50a83134f6697" [[projects]] + digest = "1:e9113641c839c21d8eaeb2c907c7276af1eddeed988df8322168c56b7e06e0e1" name = "github.com/tendermint/go-amino" packages = ["."] + pruneopts = "UT" revision = "2106ca61d91029c931fd54968c2bb02dc96b1412" version = "0.10.1" [[projects]] + digest = "1:3502c7cf4ada0e74d6a21e524fdd8ad8741b0a8da5219b9c38e3890d5ebbad1c" name = "github.com/tendermint/tendermint" packages = [ "crypto/tmhash", "libs/common", "libs/db", "libs/log", - "libs/test" + "libs/test", ] - revision = "5923b6288fe8ce9581936ee97c2bf9cf9c02c2f4" - version = "v0.22.0-rc2" + pruneopts = "UT" + revision = "5fdbcd70df57b71ffba71e1ff5f00d617852a9c0" + version = "v0.22.6" [[projects]] branch = "master" + digest = "1:ac66d893cc5a3e78b27affceb96cdcc284318ae8f8aa86f9f6f51322d7d744f5" name = "golang.org/x/crypto" packages = [ "ripemd160", - "sha3" + "sha3", ] + pruneopts = "UT" revision = "a49355c7e3f8fe157a85be2f77e6e269a0f89602" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "06b259c645a1a884f870ef81ab220a7302b156f6fc1f1500779d540a7eed0d49" + input-imports = [ + "github.com/stretchr/testify/assert", + "github.com/stretchr/testify/require", + "github.com/tendermint/go-amino", + "github.com/tendermint/tendermint/crypto/tmhash", + "github.com/tendermint/tendermint/libs/common", + "github.com/tendermint/tendermint/libs/db", + "github.com/tendermint/tendermint/libs/test", + "golang.org/x/crypto/ripemd160", + "golang.org/x/crypto/sha3", + ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index b00a0c141..39510f840 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -7,7 +7,7 @@ name = "github.com/tendermint/go-amino" [[constraint]] - version = "=0.22.0-rc2" + version = "^0.22.0" name = "github.com/tendermint/tendermint" [prune] From c917ed2043a0e0cc01425c73d16e83571d5c76b5 Mon Sep 17 00:00:00 2001 From: Mikhail Gusarov Date: Mon, 6 Aug 2018 10:44:27 +0200 Subject: [PATCH 06/10] Remove unused variable --- proof_range.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/proof_range.go b/proof_range.go index cc12618f9..2062b3fbd 100644 --- a/proof_range.go +++ b/proof_range.go @@ -365,7 +365,6 @@ func (t *Tree) getRangeProof(keyStart, keyEnd []byte, limit int) (*RangeProof, [ // nolint var innersq = []PathToLeaf(nil) var inners = PathToLeaf(nil) - var lastDepth uint8 = 0 var leafCount = 1 // from left above. var pathCount = 0 // var keys, values [][]byte defined as function outs. @@ -435,7 +434,6 @@ func (t *Tree) getRangeProof(keyStart, keyEnd []byte, limit int) (*RangeProof, [ }) } } - lastDepth = depth return false }, ) From d4ff72cbfa2e3409a85bb47341a0b64eae2f3da5 Mon Sep 17 00:00:00 2001 From: Mikhail Gusarov Date: Mon, 6 Aug 2018 10:44:39 +0200 Subject: [PATCH 07/10] Remove unreachable code --- util.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/util.go b/util.go index 96f754189..7ef19462b 100644 --- a/util.go +++ b/util.go @@ -71,8 +71,6 @@ func cpIncr(bz []byte) (ret []byte) { ret[i] = byte(0x00) if i == 0 { return append(ret, 0x00) - // Overflow - return nil } } return []byte{0x00} From da94e6c675c8b150a808ca61d0553e5f9667f222 Mon Sep 17 00:00:00 2001 From: Jeremiah Andrews Date: Sat, 11 Aug 2018 12:57:31 -0700 Subject: [PATCH 08/10] Mutable/Immutable refactor and GetImmutable snapshots (#92) * Move orphaningTree logic into VersionedTree * Move Tree.Set and Node.set to VersionedTree, fix tests * Move Tree.Remove and node.remove to VersionedTree * Rename VersionedTree/Tree to MutableTree/ImmutableTree * Rename files to match type names * Add GetImmutable and lazy loading of MutableTree * Move balance and rotate from Node to MutableTree * Add benchmarks with -benchmem, remove incomplete benchmark * Rename variables in rotation functions * Add test to check for consistent tracking of orphans * Add back ability to load old versions with idempotent saves * Add idempotent save test case --- basic_test.go | 35 +- benchmarks/bench_test.go | 18 +- .../digial-ocean-32gb-18.04-a3d16d8.txt | 57 --- ...al-ocean-64gb-fullbench-memory-8f19f23.txt | 149 ++++++ ...al-ocean-64gb-fullbench-memory-c1f6d4e.txt | 149 ++++++ doc.go | 4 +- tree.go => immutable_tree.go | 99 +--- mutable_tree.go | 469 ++++++++++++++++++ node.go | 213 +------- nodedb.go | 4 + orphaning_tree.go | 74 --- proof.go | 4 +- proof_range.go | 24 +- proof_test.go | 15 +- testutils_test.go | 6 +- tree_dotgraph.go | 2 +- tree_dotgraph_test.go | 6 +- tree_fuzz_test.go | 8 +- tree_test.go | 152 ++++-- util.go | 2 +- versioned_tree.go | 197 -------- 21 files changed, 988 insertions(+), 699 deletions(-) delete mode 100644 benchmarks/results/digial-ocean-32gb-18.04-a3d16d8.txt create mode 100644 benchmarks/results/digital-ocean-64gb-fullbench-memory-8f19f23.txt create mode 100644 benchmarks/results/digital-ocean-64gb-fullbench-memory-c1f6d4e.txt rename tree.go => immutable_tree.go (55%) create mode 100644 mutable_tree.go delete mode 100644 orphaning_tree.go delete mode 100644 versioned_tree.go diff --git a/basic_test.go b/basic_test.go index 5dca7cb20..07e4cb9fa 100644 --- a/basic_test.go +++ b/basic_test.go @@ -12,7 +12,7 @@ import ( ) func TestBasic(t *testing.T) { - tree := NewTree(nil, 0) + tree := NewMutableTree(db.NewMemDB(), 0) up := tree.Set([]byte("1"), []byte("one")) if up { t.Error("Did not expect an update (should have been create)") @@ -103,7 +103,7 @@ func TestBasic(t *testing.T) { func TestUnit(t *testing.T) { - expectHash := func(tree *Tree, hashCount int64) { + expectHash := func(tree *ImmutableTree, hashCount int64) { // ensure number of new hash calculations is as expected. hash, count := tree.hashWithCount() if count != hashCount { @@ -121,7 +121,7 @@ func TestUnit(t *testing.T) { } } - expectSet := func(tree *Tree, i int, repr string, hashCount int64) { + expectSet := func(tree *MutableTree, i int, repr string, hashCount int64) { origNode := tree.root updated := tree.Set(i2b(i), []byte{}) // ensure node was added & structure is as expected. @@ -130,11 +130,11 @@ func TestUnit(t *testing.T) { i, P(origNode), repr, P(tree.root), updated) } // ensure hash calculation requirements - expectHash(tree, hashCount) + expectHash(tree.ImmutableTree, hashCount) tree.root = origNode } - expectRemove := func(tree *Tree, i int, repr string, hashCount int64) { + expectRemove := func(tree *MutableTree, i int, repr string, hashCount int64) { origNode := tree.root value, removed := tree.Remove(i2b(i)) // ensure node was added & structure is as expected. @@ -143,7 +143,7 @@ func TestUnit(t *testing.T) { i, P(origNode), repr, P(tree.root), value, removed) } // ensure hash calculation requirements - expectHash(tree, hashCount) + expectHash(tree.ImmutableTree, hashCount) tree.root = origNode } @@ -190,7 +190,7 @@ func TestRemove(t *testing.T) { d := db.NewDB("test", "memdb", "") defer d.Close() - t1 := NewVersionedTree(d, size) + t1 := NewMutableTree(d, size) // insert a bunch of random nodes keys := make([][]byte, size) @@ -220,7 +220,7 @@ func TestIntegration(t *testing.T) { } records := make([]*record, 400) - tree := NewTree(nil, 0) + tree := NewMutableTree(db.NewMemDB(), 0) randomRecord := func() *record { return &record{randstr(20), randstr(20)} @@ -302,7 +302,7 @@ func TestIterateRange(t *testing.T) { } sort.Strings(keys) - tree := NewTree(nil, 0) + tree := NewMutableTree(db.NewMemDB(), 0) // insert all the data for _, r := range records { @@ -372,14 +372,14 @@ func TestPersistence(t *testing.T) { } // Construct some tree and save it - t1 := NewVersionedTree(db, 0) + t1 := NewMutableTree(db, 0) for key, value := range records { t1.Set([]byte(key), []byte(value)) } t1.SaveVersion() // Load a tree - t2 := NewVersionedTree(db, 0) + t2 := NewMutableTree(db, 0) t2.Load() for key, value := range records { _, t2value := t2.Get64([]byte(key)) @@ -390,12 +390,11 @@ func TestPersistence(t *testing.T) { } func TestProof(t *testing.T) { - t.Skipf("This test has a race condition causing it to occasionally panic.") // Construct some random tree db := db.NewMemDB() - tree := NewVersionedTree(db, 100) - for i := 0; i < 1000; i++ { + tree := NewMutableTree(db, 100) + for i := 0; i < 10; i++ { key, value := randstr(20), randstr(20) tree.Set([]byte(key), []byte(value)) } @@ -404,7 +403,7 @@ func TestProof(t *testing.T) { tree.SaveVersion() // Add more items so it's not all persisted - for i := 0; i < 100; i++ { + for i := 0; i < 10; i++ { key, value := randstr(20), randstr(20) tree.Set([]byte(key), []byte(value)) } @@ -415,7 +414,7 @@ func TestProof(t *testing.T) { assert.NoError(t, err) assert.Equal(t, value, value2) if assert.NotNil(t, proof) { - verifyProof(t, proof, tree.Hash()) + verifyProof(t, proof, tree.WorkingHash()) } return false }) @@ -423,7 +422,7 @@ func TestProof(t *testing.T) { func TestTreeProof(t *testing.T) { db := db.NewMemDB() - tree := NewTree(db, 100) + tree := NewMutableTree(db, 100) // should get false for proof with nil root _, _, err := tree.GetWithProof([]byte("foo")) @@ -442,7 +441,7 @@ func TestTreeProof(t *testing.T) { assert.NoError(t, err) // valid proof for real keys - root := tree.Hash() + root := tree.WorkingHash() for _, key := range keys { value, proof, err := tree.GetWithProof(key) if assert.NoError(t, err) { diff --git a/benchmarks/bench_test.go b/benchmarks/bench_test.go index 820ba8db3..9284a7a2e 100644 --- a/benchmarks/bench_test.go +++ b/benchmarks/bench_test.go @@ -20,8 +20,8 @@ func randBytes(length int) []byte { return key } -func prepareTree(b *testing.B, db db.DB, size, keyLen, dataLen int) (*iavl.VersionedTree, [][]byte) { - t := iavl.NewVersionedTree(db, size) +func prepareTree(b *testing.B, db db.DB, size, keyLen, dataLen int) (*iavl.MutableTree, [][]byte) { + t := iavl.NewMutableTree(db, size) keys := make([][]byte, size) for i := 0; i < size; i++ { @@ -35,7 +35,7 @@ func prepareTree(b *testing.B, db db.DB, size, keyLen, dataLen int) (*iavl.Versi } // commit tree saves a new version and deletes and old one... -func commitTree(b *testing.B, t *iavl.VersionedTree) { +func commitTree(b *testing.B, t *iavl.MutableTree) { t.Hash() _, version, err := t.SaveVersion() if err != nil { @@ -49,14 +49,14 @@ func commitTree(b *testing.B, t *iavl.VersionedTree) { } } -func runQueries(b *testing.B, t *iavl.VersionedTree, keyLen int) { +func runQueries(b *testing.B, t *iavl.MutableTree, keyLen int) { for i := 0; i < b.N; i++ { q := randBytes(keyLen) t.Get(q) } } -func runKnownQueries(b *testing.B, t *iavl.VersionedTree, keys [][]byte) { +func runKnownQueries(b *testing.B, t *iavl.MutableTree, keys [][]byte) { l := int32(len(keys)) for i := 0; i < b.N; i++ { q := keys[rand.Int31n(l)] @@ -64,7 +64,7 @@ func runKnownQueries(b *testing.B, t *iavl.VersionedTree, keys [][]byte) { } } -func runInsert(b *testing.B, t *iavl.VersionedTree, keyLen, dataLen, blockSize int) *iavl.VersionedTree { +func runInsert(b *testing.B, t *iavl.MutableTree, keyLen, dataLen, blockSize int) *iavl.MutableTree { for i := 1; i <= b.N; i++ { t.Set(randBytes(keyLen), randBytes(dataLen)) if i%blockSize == 0 { @@ -75,7 +75,7 @@ func runInsert(b *testing.B, t *iavl.VersionedTree, keyLen, dataLen, blockSize i return t } -func runUpdate(b *testing.B, t *iavl.VersionedTree, dataLen, blockSize int, keys [][]byte) *iavl.VersionedTree { +func runUpdate(b *testing.B, t *iavl.MutableTree, dataLen, blockSize int, keys [][]byte) *iavl.MutableTree { l := int32(len(keys)) for i := 1; i <= b.N; i++ { key := keys[rand.Int31n(l)] @@ -87,7 +87,7 @@ func runUpdate(b *testing.B, t *iavl.VersionedTree, dataLen, blockSize int, keys return t } -func runDelete(b *testing.B, t *iavl.VersionedTree, blockSize int, keys [][]byte) *iavl.VersionedTree { +func runDelete(b *testing.B, t *iavl.MutableTree, blockSize int, keys [][]byte) *iavl.MutableTree { var key []byte l := int32(len(keys)) for i := 1; i <= b.N; i++ { @@ -103,7 +103,7 @@ func runDelete(b *testing.B, t *iavl.VersionedTree, blockSize int, keys [][]byte } // runBlock measures time for an entire block, not just one tx -func runBlock(b *testing.B, t *iavl.VersionedTree, keyLen, dataLen, blockSize int, keys [][]byte) *iavl.VersionedTree { +func runBlock(b *testing.B, t *iavl.MutableTree, keyLen, dataLen, blockSize int, keys [][]byte) *iavl.MutableTree { l := int32(len(keys)) // XXX: This was adapted to work with VersionedTree but needs to be re-thought. diff --git a/benchmarks/results/digial-ocean-32gb-18.04-a3d16d8.txt b/benchmarks/results/digial-ocean-32gb-18.04-a3d16d8.txt deleted file mode 100644 index 095a2540d..000000000 --- a/benchmarks/results/digial-ocean-32gb-18.04-a3d16d8.txt +++ /dev/null @@ -1,57 +0,0 @@ -cd benchmarks && \ - go test -bench=RandomBytes . && \ - go test -bench=Small . && \ - go test -bench=Medium . && \ - go test -bench=BenchmarkMemKeySizes . -goos: linux -goarch: amd64 -pkg: github.com/tendermint/iavl/benchmarks -BenchmarkRandomBytes/random-4-8 10000000 125 ns/op -BenchmarkRandomBytes/random-16-8 10000000 199 ns/op -BenchmarkRandomBytes/random-32-8 5000000 273 ns/op -BenchmarkRandomBytes/random-100-8 2000000 619 ns/op -BenchmarkRandomBytes/random-1000-8 300000 4655 ns/op -PASS -ok github.com/tendermint/iavl/benchmarks 8.571s -Init Tree took 0.90 MB -goos: linux -goarch: amd64 -pkg: github.com/tendermint/iavl/benchmarks -BenchmarkSmall/memdb-1000-100-4-10/query-miss-8 200000 7393 ns/op -BenchmarkSmall/memdb-1000-100-4-10/query-hits-8 200000 9513 ns/op -BenchmarkSmall/memdb-1000-100-4-10/update-8 10000 274623 ns/op -BenchmarkSmall/memdb-1000-100-4-10/block-8 50 39110192 ns/op -Init Tree took 0.49 MB -BenchmarkSmall/goleveldb-1000-100-4-10/query-miss-8 200000 10313 ns/op -BenchmarkSmall/goleveldb-1000-100-4-10/query-hits-8 100000 14936 ns/op -BenchmarkSmall/goleveldb-1000-100-4-10/update-8 10000 173268 ns/op -BenchmarkSmall/goleveldb-1000-100-4-10/block-8 100 30316707 ns/op -Init Tree took 0.49 MB -BenchmarkSmall/leveldb-1000-100-4-10/query-miss-8 200000 9815 ns/op -BenchmarkSmall/leveldb-1000-100-4-10/query-hits-8 100000 13164 ns/op -BenchmarkSmall/leveldb-1000-100-4-10/update-8 10000 167816 ns/op -BenchmarkSmall/leveldb-1000-100-4-10/block-8 100 27400294 ns/op -PASS -ok github.com/tendermint/iavl/benchmarks 25.039s -Init Tree took 85.10 MB -goos: linux -goarch: amd64 -pkg: github.com/tendermint/iavl/benchmarks -BenchmarkMedium/memdb-100000-100-16-40/query-miss-8 100000 16066 ns/op -BenchmarkMedium/memdb-100000-100-16-40/query-hits-8 100000 16443 ns/op -BenchmarkMedium/memdb-100000-100-16-40/update-8 2000 1349641 ns/op -BenchmarkMedium/memdb-100000-100-16-40/block-8 5 275106058 ns/op -Init Tree took 47.22 MB -BenchmarkMedium/goleveldb-100000-100-16-40/query-miss-8 30000 38140 ns/op -BenchmarkMedium/goleveldb-100000-100-16-40/query-hits-8 30000 42500 ns/op -BenchmarkMedium/goleveldb-100000-100-16-40/update-8 10000 444647 ns/op -BenchmarkMedium/goleveldb-100000-100-16-40/block-8 30 57406398 ns/op -Init Tree took 43.70 MB -BenchmarkMedium/leveldb-100000-100-16-40/query-miss-8 30000 38214 ns/op -BenchmarkMedium/leveldb-100000-100-16-40/query-hits-8 30000 40582 ns/op -BenchmarkMedium/leveldb-100000-100-16-40/update-8 10000 498158 ns/op -BenchmarkMedium/leveldb-100000-100-16-40/block-8 30 59098590 ns/op -PASS -ok github.com/tendermint/iavl/benchmarks 49.686s -PASS -ok github.com/tendermint/iavl/benchmarks 0.008s diff --git a/benchmarks/results/digital-ocean-64gb-fullbench-memory-8f19f23.txt b/benchmarks/results/digital-ocean-64gb-fullbench-memory-8f19f23.txt new file mode 100644 index 000000000..ca6a7c618 --- /dev/null +++ b/benchmarks/results/digital-ocean-64gb-fullbench-memory-8f19f23.txt @@ -0,0 +1,149 @@ +cd benchmarks && \ + go test -bench=RandomBytes . -benchmem && \ + go test -bench=Small . -benchmem && \ + go test -bench=Medium . -benchmem && \ + go test -timeout=30m -bench=Large . -benchmem && \ + go test -bench=Mem . -benchmem && \ + go test -timeout=60m -bench=LevelDB . -benchmem +goos: linux +goarch: amd64 +pkg: github.com/tendermint/iavl/benchmarks +BenchmarkRandomBytes/random-4-24 20000000 72.0 ns/op 4 B/op 1 allocs/op +BenchmarkRandomBytes/random-16-24 20000000 118 ns/op 16 B/op 1 allocs/op +BenchmarkRandomBytes/random-32-24 10000000 156 ns/op 32 B/op 1 allocs/op +BenchmarkRandomBytes/random-100-24 5000000 370 ns/op 112 B/op 1 allocs/op +BenchmarkRandomBytes/random-1000-24 500000 2958 ns/op 1024 B/op 1 allocs/op +PASS +ok github.com/tendermint/iavl/benchmarks 9.506s +Init Tree took 0.91 MB +goos: linux +goarch: amd64 +pkg: github.com/tendermint/iavl/benchmarks +BenchmarkSmall/memdb-1000-100-4-10/query-miss-24 500000 4420 ns/op 435 B/op 9 allocs/op +BenchmarkSmall/memdb-1000-100-4-10/query-hits-24 200000 5272 ns/op 634 B/op 12 allocs/op +BenchmarkSmall/memdb-1000-100-4-10/update-24 10000 159192 ns/op 42771 B/op 764 allocs/op +BenchmarkSmall/memdb-1000-100-4-10/block-24 100 24824001 ns/op 6620344 B/op 120912 allocs/op +Init Tree took 0.49 MB +BenchmarkSmall/goleveldb-1000-100-4-10/query-miss-24 300000 6368 ns/op 649 B/op 13 allocs/op +BenchmarkSmall/goleveldb-1000-100-4-10/query-hits-24 200000 8741 ns/op 918 B/op 18 allocs/op +BenchmarkSmall/goleveldb-1000-100-4-10/update-24 10000 109113 ns/op 22326 B/op 254 allocs/op +BenchmarkSmall/goleveldb-1000-100-4-10/block-24 100 17755475 ns/op 3487423 B/op 39158 allocs/op +Init Tree took 0.49 MB +BenchmarkSmall/leveldb-1000-100-4-10/query-miss-24 300000 6584 ns/op 651 B/op 14 allocs/op +BenchmarkSmall/leveldb-1000-100-4-10/query-hits-24 200000 7898 ns/op 918 B/op 18 allocs/op +BenchmarkSmall/leveldb-1000-100-4-10/update-24 10000 102305 ns/op 22352 B/op 254 allocs/op +BenchmarkSmall/leveldb-1000-100-4-10/block-24 100 16187510 ns/op 3461634 B/op 39097 allocs/op +PASS +ok github.com/tendermint/iavl/benchmarks 20.641s +Init Tree took 85.10 MB +goos: linux +goarch: amd64 +pkg: github.com/tendermint/iavl/benchmarks +BenchmarkMedium/memdb-100000-100-16-40/query-miss-24 200000 9595 ns/op 513 B/op 10 allocs/op +BenchmarkMedium/memdb-100000-100-16-40/query-hits-24 200000 10834 ns/op 676 B/op 12 allocs/op +BenchmarkMedium/memdb-100000-100-16-40/update-24 3000 1261394 ns/op 246830 B/op 4746 allocs/op +BenchmarkMedium/memdb-100000-100-16-40/block-24 10 198937291 ns/op 40016809 B/op 795945 allocs/op +Init Tree took 47.42 MB +BenchmarkMedium/goleveldb-100000-100-16-40/query-miss-24 50000 22673 ns/op 1594 B/op 27 allocs/op +BenchmarkMedium/goleveldb-100000-100-16-40/query-hits-24 50000 28135 ns/op 2144 B/op 35 allocs/op +BenchmarkMedium/goleveldb-100000-100-16-40/update-24 10000 294852 ns/op 53115 B/op 592 allocs/op +BenchmarkMedium/goleveldb-100000-100-16-40/block-24 50 34807659 ns/op 5968256 B/op 67622 allocs/op +Init Tree took 47.80 MB +BenchmarkMedium/leveldb-100000-100-16-40/query-miss-24 50000 22686 ns/op 1532 B/op 25 allocs/op +BenchmarkMedium/leveldb-100000-100-16-40/query-hits-24 50000 27668 ns/op 2159 B/op 35 allocs/op +BenchmarkMedium/leveldb-100000-100-16-40/update-24 10000 287108 ns/op 53026 B/op 593 allocs/op +BenchmarkMedium/leveldb-100000-100-16-40/block-24 30 35596044 ns/op 6206475 B/op 67118 allocs/op +PASS +ok github.com/tendermint/iavl/benchmarks 38.966s +Init Tree took 917.91 MB +goos: linux +goarch: amd64 +pkg: github.com/tendermint/iavl/benchmarks +BenchmarkLarge/memdb-1000000-100-16-40/query-miss-24 100000 15911 ns/op 1061 B/op 20 allocs/op +BenchmarkLarge/memdb-1000000-100-16-40/query-hits-24 100000 15748 ns/op 829 B/op 15 allocs/op +BenchmarkLarge/memdb-1000000-100-16-40/update-24 300 5161409 ns/op 994056 B/op 20570 allocs/op +BenchmarkLarge/memdb-1000000-100-16-40/block-24 2 514923704 ns/op 100084344 B/op 2069282 allocs/op +Init Tree took 416.94 MB +BenchmarkLarge/goleveldb-1000000-100-16-40/query-miss-24 20000 60866 ns/op 4902 B/op 82 allocs/op +BenchmarkLarge/goleveldb-1000000-100-16-40/query-hits-24 30000 49123 ns/op 3745 B/op 62 allocs/op +BenchmarkLarge/goleveldb-1000000-100-16-40/update-24 10000 478900 ns/op 81376 B/op 836 allocs/op +BenchmarkLarge/goleveldb-1000000-100-16-40/block-24 20 53327657 ns/op 10397199 B/op 98883 allocs/op +Init Tree took 404.64 MB +BenchmarkLarge/leveldb-1000000-100-16-40/query-miss-24 20000 57290 ns/op 4590 B/op 76 allocs/op +BenchmarkLarge/leveldb-1000000-100-16-40/query-hits-24 30000 48691 ns/op 3640 B/op 60 allocs/op +BenchmarkLarge/leveldb-1000000-100-16-40/update-24 5000 381530 ns/op 69914 B/op 736 allocs/op +BenchmarkLarge/leveldb-1000000-100-16-40/block-24 30 56220875 ns/op 10656291 B/op 101281 allocs/op +Init Tree took 24.87 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/query-miss-24 50000 23263 ns/op 1745 B/op 27 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/query-hits-24 50000 25201 ns/op 2238 B/op 34 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/update-24 10000 283769 ns/op 52149 B/op 563 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/block-24 50 33455540 ns/op 5749534 B/op 64612 allocs/op +Init Tree took 39.73 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/query-miss-24 50000 23683 ns/op 2716 B/op 28 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/query-hits-24 50000 32174 ns/op 3716 B/op 37 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/update-24 10000 317281 ns/op 64047 B/op 616 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/block-24 50 44231625 ns/op 7399545 B/op 73233 allocs/op +Init Tree took 264.19 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/query-miss-24 30000 35564 ns/op 11875 B/op 30 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/query-hits-24 30000 45093 ns/op 16763 B/op 39 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/update-24 10000 574978 ns/op 207200 B/op 752 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/block-24 20 76693083 ns/op 28530852 B/op 92942 allocs/op +Init Tree took 2676.96 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/query-miss-24 10000 147724 ns/op 256119 B/op 64 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/query-hits-24 10000 169279 ns/op 307666 B/op 70 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/update-24 1000 3882880 ns/op 2601694 B/op 639 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/block-24 10 520453137 ns/op 431258147 B/op 102744 allocs/op +PASS +ok github.com/tendermint/iavl/benchmarks 448.959s +PASS +ok github.com/tendermint/iavl/benchmarks 0.008s +Init Tree took 47.64 MB +goos: linux +goarch: amd64 +pkg: github.com/tendermint/iavl/benchmarks +BenchmarkLevelDBBatchSizes/goleveldb-100000-5-16-40/query-miss-24 50000 22097 ns/op 1601 B/op 27 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-5-16-40/query-hits-24 50000 27734 ns/op 2157 B/op 35 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-5-16-40/update-24 10000 511317 ns/op 78187 B/op 842 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-5-16-40/block-24 500 2681891 ns/op 398000 B/op 4350 allocs/op +Init Tree took 47.60 MB +BenchmarkLevelDBBatchSizes/goleveldb-100000-25-16-40/query-miss-24 50000 23397 ns/op 1536 B/op 26 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-25-16-40/query-hits-24 50000 28137 ns/op 2163 B/op 35 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-25-16-40/update-24 10000 349234 ns/op 64083 B/op 704 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-25-16-40/block-24 200 9405218 ns/op 1702630 B/op 18575 allocs/op +Init Tree took 35.87 MB +BenchmarkLevelDBBatchSizes/goleveldb-100000-100-16-40/query-miss-24 50000 22684 ns/op 1609 B/op 27 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-100-16-40/query-hits-24 50000 28836 ns/op 2156 B/op 35 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-100-16-40/update-24 10000 279935 ns/op 52609 B/op 595 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-100-16-40/block-24 50 34924252 ns/op 5994318 B/op 67455 allocs/op +Init Tree took 40.94 MB +BenchmarkLevelDBBatchSizes/goleveldb-100000-400-16-40/query-miss-24 50000 21974 ns/op 1568 B/op 26 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-400-16-40/query-hits-24 50000 29079 ns/op 2212 B/op 36 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-400-16-40/update-24 10000 200319 ns/op 40108 B/op 454 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-400-16-40/block-24 10 143250792 ns/op 23631405 B/op 273493 allocs/op +Init Tree took 38.19 MB +BenchmarkLevelDBBatchSizes/goleveldb-100000-2000-16-40/query-miss-24 50000 22070 ns/op 1543 B/op 26 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-2000-16-40/query-hits-24 50000 28305 ns/op 2150 B/op 35 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-2000-16-40/update-24 10000 150990 ns/op 31752 B/op 326 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-2000-16-40/block-24 3 416917935 ns/op 78008320 B/op 824207 allocs/op +Init Tree took 23.60 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/query-miss-24 100000 19841 ns/op 1591 B/op 24 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/query-hits-24 50000 25320 ns/op 2237 B/op 34 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/update-24 10000 310418 ns/op 51051 B/op 564 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/block-24 50 33174907 ns/op 5796150 B/op 64856 allocs/op +Init Tree took 39.77 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/query-miss-24 100000 23820 ns/op 2638 B/op 27 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/query-hits-24 50000 28973 ns/op 3720 B/op 37 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/update-24 10000 315650 ns/op 64815 B/op 618 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/block-24 50 41392373 ns/op 7424452 B/op 74022 allocs/op +Init Tree took 266.07 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/query-miss-24 30000 33961 ns/op 11884 B/op 30 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/query-hits-24 30000 45861 ns/op 16535 B/op 39 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/update-24 10000 594813 ns/op 213379 B/op 760 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/block-24 20 77458855 ns/op 28589214 B/op 92146 allocs/op +Init Tree took 2677.35 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/query-miss-24 10000 148305 ns/op 252620 B/op 63 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/query-hits-24 10000 169529 ns/op 310811 B/op 71 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/update-24 1000 3540659 ns/op 2632158 B/op 640 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/block-24 10 515252342 ns/op 440797695 B/op 102385 allocs/op +PASS +ok github.com/tendermint/iavl/benchmarks 243.566s diff --git a/benchmarks/results/digital-ocean-64gb-fullbench-memory-c1f6d4e.txt b/benchmarks/results/digital-ocean-64gb-fullbench-memory-c1f6d4e.txt new file mode 100644 index 000000000..5c43e9624 --- /dev/null +++ b/benchmarks/results/digital-ocean-64gb-fullbench-memory-c1f6d4e.txt @@ -0,0 +1,149 @@ +cd benchmarks && \ + go test -bench=RandomBytes . -benchmem && \ + go test -bench=Small . -benchmem && \ + go test -bench=Medium . -benchmem && \ + go test -timeout=30m -bench=Large . -benchmem && \ + go test -bench=Mem . -benchmem && \ + go test -timeout=60m -bench=LevelDB . -benchmem +goos: linux +goarch: amd64 +pkg: github.com/tendermint/iavl/benchmarks +BenchmarkRandomBytes/random-4-24 20000000 74.8 ns/op 4 B/op 1 allocs/op +BenchmarkRandomBytes/random-16-24 20000000 121 ns/op 16 B/op 1 allocs/op +BenchmarkRandomBytes/random-32-24 10000000 166 ns/op 32 B/op 1 allocs/op +BenchmarkRandomBytes/random-100-24 5000000 376 ns/op 112 B/op 1 allocs/op +BenchmarkRandomBytes/random-1000-24 500000 2943 ns/op 1024 B/op 1 allocs/op +PASS +ok github.com/tendermint/iavl/benchmarks 9.769s +Init Tree took 0.91 MB +goos: linux +goarch: amd64 +pkg: github.com/tendermint/iavl/benchmarks +BenchmarkSmall/memdb-1000-100-4-10/query-miss-24 300000 4081 ns/op 434 B/op 9 allocs/op +BenchmarkSmall/memdb-1000-100-4-10/query-hits-24 300000 5112 ns/op 633 B/op 12 allocs/op +BenchmarkSmall/memdb-1000-100-4-10/update-24 10000 157089 ns/op 42683 B/op 763 allocs/op +BenchmarkSmall/memdb-1000-100-4-10/block-24 100 25654741 ns/op 6619026 B/op 120940 allocs/op +Init Tree took 0.49 MB +BenchmarkSmall/goleveldb-1000-100-4-10/query-miss-24 200000 7127 ns/op 637 B/op 13 allocs/op +BenchmarkSmall/goleveldb-1000-100-4-10/query-hits-24 200000 8713 ns/op 918 B/op 18 allocs/op +BenchmarkSmall/goleveldb-1000-100-4-10/update-24 10000 103509 ns/op 22232 B/op 253 allocs/op +BenchmarkSmall/goleveldb-1000-100-4-10/block-24 100 17394312 ns/op 3483478 B/op 39338 allocs/op +Init Tree took 0.49 MB +BenchmarkSmall/leveldb-1000-100-4-10/query-miss-24 300000 6214 ns/op 646 B/op 13 allocs/op +BenchmarkSmall/leveldb-1000-100-4-10/query-hits-24 200000 8254 ns/op 919 B/op 18 allocs/op +BenchmarkSmall/leveldb-1000-100-4-10/update-24 10000 107058 ns/op 22312 B/op 254 allocs/op +BenchmarkSmall/leveldb-1000-100-4-10/block-24 100 17031744 ns/op 3482495 B/op 39144 allocs/op +PASS +ok github.com/tendermint/iavl/benchmarks 19.874s +Init Tree took 85.10 MB +goos: linux +goarch: amd64 +pkg: github.com/tendermint/iavl/benchmarks +BenchmarkMedium/memdb-100000-100-16-40/query-miss-24 200000 10064 ns/op 513 B/op 10 allocs/op +BenchmarkMedium/memdb-100000-100-16-40/query-hits-24 200000 11143 ns/op 676 B/op 12 allocs/op +BenchmarkMedium/memdb-100000-100-16-40/update-24 3000 1303374 ns/op 246834 B/op 4746 allocs/op +BenchmarkMedium/memdb-100000-100-16-40/block-24 10 190258294 ns/op 40016520 B/op 795943 allocs/op +Init Tree took 47.63 MB +BenchmarkMedium/goleveldb-100000-100-16-40/query-miss-24 50000 22452 ns/op 1539 B/op 26 allocs/op +BenchmarkMedium/goleveldb-100000-100-16-40/query-hits-24 50000 28301 ns/op 2148 B/op 35 allocs/op +BenchmarkMedium/goleveldb-100000-100-16-40/update-24 10000 296013 ns/op 52887 B/op 594 allocs/op +BenchmarkMedium/goleveldb-100000-100-16-40/block-24 30 35855483 ns/op 6213133 B/op 67658 allocs/op +Init Tree took 42.26 MB +BenchmarkMedium/leveldb-100000-100-16-40/query-miss-24 50000 22802 ns/op 1595 B/op 27 allocs/op +BenchmarkMedium/leveldb-100000-100-16-40/query-hits-24 50000 31757 ns/op 2147 B/op 35 allocs/op +BenchmarkMedium/leveldb-100000-100-16-40/update-24 10000 297615 ns/op 52713 B/op 594 allocs/op +BenchmarkMedium/leveldb-100000-100-16-40/block-24 30 36791150 ns/op 6289507 B/op 67963 allocs/op +PASS +ok github.com/tendermint/iavl/benchmarks 38.736s +Init Tree took 917.92 MB +goos: linux +goarch: amd64 +pkg: github.com/tendermint/iavl/benchmarks +BenchmarkLarge/memdb-1000000-100-16-40/query-miss-24 100000 15781 ns/op 1061 B/op 20 allocs/op +BenchmarkLarge/memdb-1000000-100-16-40/query-hits-24 100000 15750 ns/op 829 B/op 15 allocs/op +BenchmarkLarge/memdb-1000000-100-16-40/update-24 300 5256647 ns/op 994044 B/op 20570 allocs/op +BenchmarkLarge/memdb-1000000-100-16-40/block-24 2 534785952 ns/op 100083320 B/op 2069277 allocs/op +Init Tree took 416.96 MB +BenchmarkLarge/goleveldb-1000000-100-16-40/query-miss-24 20000 59997 ns/op 4900 B/op 82 allocs/op +BenchmarkLarge/goleveldb-1000000-100-16-40/query-hits-24 30000 51637 ns/op 3748 B/op 62 allocs/op +BenchmarkLarge/goleveldb-1000000-100-16-40/update-24 10000 476117 ns/op 81887 B/op 838 allocs/op +BenchmarkLarge/goleveldb-1000000-100-16-40/block-24 30 56340657 ns/op 10034120 B/op 95529 allocs/op +Init Tree took 404.27 MB +BenchmarkLarge/leveldb-1000000-100-16-40/query-miss-24 20000 62528 ns/op 5003 B/op 81 allocs/op +BenchmarkLarge/leveldb-1000000-100-16-40/query-hits-24 30000 50966 ns/op 3701 B/op 61 allocs/op +BenchmarkLarge/leveldb-1000000-100-16-40/update-24 10000 456299 ns/op 86644 B/op 841 allocs/op +BenchmarkLarge/leveldb-1000000-100-16-40/block-24 30 58929008 ns/op 12193146 B/op 100887 allocs/op +Init Tree took 25.20 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/query-miss-24 50000 20237 ns/op 1659 B/op 26 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/query-hits-24 50000 26440 ns/op 2248 B/op 34 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/update-24 10000 276849 ns/op 52649 B/op 565 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/block-24 50 33371134 ns/op 5881967 B/op 65264 allocs/op +Init Tree took 39.72 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/query-miss-24 50000 23127 ns/op 2732 B/op 28 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/query-hits-24 50000 30518 ns/op 3739 B/op 37 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/update-24 10000 317968 ns/op 63822 B/op 616 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/block-24 50 40372117 ns/op 7424951 B/op 73717 allocs/op +Init Tree took 264.06 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/query-miss-24 30000 34792 ns/op 11953 B/op 30 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/query-hits-24 30000 45317 ns/op 16693 B/op 39 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/update-24 5000 521571 ns/op 189390 B/op 695 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/block-24 20 78074233 ns/op 26628664 B/op 93850 allocs/op +Init Tree took 2676.68 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/query-miss-24 10000 160833 ns/op 257161 B/op 65 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/query-hits-24 10000 172494 ns/op 312459 B/op 71 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/update-24 1000 3285298 ns/op 2404346 B/op 621 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/block-24 5 482757364 ns/op 364325902 B/op 84213 allocs/op +PASS +ok github.com/tendermint/iavl/benchmarks 441.311s +PASS +ok github.com/tendermint/iavl/benchmarks 0.008s +Init Tree took 47.04 MB +goos: linux +goarch: amd64 +pkg: github.com/tendermint/iavl/benchmarks +BenchmarkLevelDBBatchSizes/goleveldb-100000-5-16-40/query-miss-24 50000 22448 ns/op 1530 B/op 25 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-5-16-40/query-hits-24 50000 32738 ns/op 2172 B/op 35 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-5-16-40/update-24 10000 515236 ns/op 78273 B/op 843 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-5-16-40/block-24 500 2702588 ns/op 402931 B/op 4385 allocs/op +Init Tree took 47.13 MB +BenchmarkLevelDBBatchSizes/goleveldb-100000-25-16-40/query-miss-24 50000 23648 ns/op 1598 B/op 27 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-25-16-40/query-hits-24 50000 28103 ns/op 2160 B/op 35 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-25-16-40/update-24 10000 351358 ns/op 65065 B/op 703 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-25-16-40/block-24 200 9918965 ns/op 1711946 B/op 18689 allocs/op +Init Tree took 42.02 MB +BenchmarkLevelDBBatchSizes/goleveldb-100000-100-16-40/query-miss-24 50000 24165 ns/op 1620 B/op 27 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-100-16-40/query-hits-24 50000 29367 ns/op 2154 B/op 35 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-100-16-40/update-24 10000 291198 ns/op 53061 B/op 594 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-100-16-40/block-24 30 35941442 ns/op 6034383 B/op 67049 allocs/op +Init Tree took 45.32 MB +BenchmarkLevelDBBatchSizes/goleveldb-100000-400-16-40/query-miss-24 50000 22196 ns/op 1579 B/op 26 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-400-16-40/query-hits-24 50000 28725 ns/op 2154 B/op 35 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-400-16-40/update-24 10000 200024 ns/op 40108 B/op 448 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-400-16-40/block-24 10 152324790 ns/op 23616529 B/op 273242 allocs/op +Init Tree took 38.47 MB +BenchmarkLevelDBBatchSizes/goleveldb-100000-2000-16-40/query-miss-24 50000 22555 ns/op 1538 B/op 26 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-2000-16-40/query-hits-24 50000 28427 ns/op 2159 B/op 35 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-2000-16-40/update-24 10000 143810 ns/op 30211 B/op 320 allocs/op +BenchmarkLevelDBBatchSizes/goleveldb-100000-2000-16-40/block-24 3 408154340 ns/op 78403509 B/op 815927 allocs/op +Init Tree took 27.54 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/query-miss-24 50000 20109 ns/op 1668 B/op 26 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/query-hits-24 50000 25620 ns/op 2243 B/op 34 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/update-24 10000 276745 ns/op 52750 B/op 566 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100/block-24 50 33831408 ns/op 5826982 B/op 64847 allocs/op +Init Tree took 39.80 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/query-miss-24 50000 23057 ns/op 2722 B/op 28 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/query-hits-24 50000 28850 ns/op 3742 B/op 37 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/update-24 10000 310652 ns/op 63335 B/op 617 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-1000/block-24 30 40865126 ns/op 7547879 B/op 74541 allocs/op +Init Tree took 266.72 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/query-miss-24 30000 33848 ns/op 12084 B/op 30 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/query-hits-24 30000 43692 ns/op 16592 B/op 39 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/update-24 10000 588636 ns/op 211307 B/op 754 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-10000/block-24 20 82922740 ns/op 29007420 B/op 93883 allocs/op +Init Tree took 2675.64 MB +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/query-miss-24 10000 147862 ns/op 250440 B/op 63 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/query-hits-24 10000 178420 ns/op 313624 B/op 72 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/update-24 1000 3606703 ns/op 2569932 B/op 640 allocs/op +BenchmarkLevelDBLargeData/goleveldb-50000-100-32-100000/block-24 5 459105626 ns/op 401755606 B/op 89339 allocs/op +PASS +ok github.com/tendermint/iavl/benchmarks 241.593s diff --git a/doc.go b/doc.go index 7e4891bcb..7751bccad 100644 --- a/doc.go +++ b/doc.go @@ -2,13 +2,13 @@ // for persisting key-value pairs. // // -// Basic usage of VersionedTree. +// Basic usage of MutableTree. // // import "github.com/tendermint/iavl" // import "github.com/tendermint/tendermint/libs/db" // ... // -// tree := iavl.NewVersionedTree(db.NewMemDB(), 128) +// tree := iavl.NewMutableTree(db.NewMemDB(), 128) // // tree.IsEmpty() // true // diff --git a/tree.go b/immutable_tree.go similarity index 55% rename from tree.go rename to immutable_tree.go index c92901532..e334ef52b 100644 --- a/tree.go +++ b/immutable_tree.go @@ -7,29 +7,29 @@ import ( dbm "github.com/tendermint/tendermint/libs/db" ) -// Tree is a container for an immutable AVL+ Tree. Changes are performed by +// ImmutableTree is a container for an immutable AVL+ ImmutableTree. Changes are performed by // swapping the internal root with a new one, while the container is mutable. // Note that this tree is not thread-safe. -type Tree struct { +type ImmutableTree struct { root *Node ndb *nodeDB version int64 } -// NewTree creates both in-memory and persistent instances -func NewTree(db dbm.DB, cacheSize int) *Tree { +// NewImmutableTree creates both in-memory and persistent instances +func NewImmutableTree(db dbm.DB, cacheSize int) *ImmutableTree { if db == nil { // In-memory Tree. - return &Tree{} + return &ImmutableTree{} } - return &Tree{ + return &ImmutableTree{ // NodeDB-backed Tree. ndb: newNodeDB(db, cacheSize), } } // String returns a string representation of Tree. -func (t *Tree) String() string { +func (t *ImmutableTree) String() string { leaves := []string{} t.Iterate(func(key []byte, val []byte) (stop bool) { leaves = append(leaves, fmt.Sprintf("%x: %x", key, val)) @@ -39,11 +39,11 @@ func (t *Tree) String() string { } // Size returns the number of leaf nodes in the tree. -func (t *Tree) Size() int { +func (t *ImmutableTree) Size() int { return int(t.Size64()) } -func (t *Tree) Size64() int64 { +func (t *ImmutableTree) Size64() int64 { if t.root == nil { return 0 } @@ -51,20 +51,20 @@ func (t *Tree) Size64() int64 { } // Version returns the version of the tree. -func (t *Tree) Version() int { +func (t *ImmutableTree) Version() int { return int(t.Version64()) } -func (t *Tree) Version64() int64 { +func (t *ImmutableTree) Version64() int64 { return t.version } // Height returns the height of the tree. -func (t *Tree) Height() int { +func (t *ImmutableTree) Height() int { return int(t.Height8()) } -func (t *Tree) Height8() int8 { +func (t *ImmutableTree) Height8() int8 { if t.root == nil { return 0 } @@ -72,34 +72,15 @@ func (t *Tree) Height8() int8 { } // Has returns whether or not a key exists. -func (t *Tree) Has(key []byte) bool { +func (t *ImmutableTree) Has(key []byte) bool { if t.root == nil { return false } return t.root.has(t, key) } -// Set a key. Nil values are not supported. -func (t *Tree) Set(key []byte, value []byte) (updated bool) { - _, updated = t.set(key, value) - return updated -} - -func (t *Tree) set(key []byte, value []byte) (orphaned []*Node, updated bool) { - if value == nil { - panic(fmt.Sprintf("Attempt to store nil value at key '%s'", key)) - } - if t.root == nil { - t.root = NewNode(key, value, t.version+1) - return nil, false - } - t.root, updated, orphaned = t.root.set(t, key, value) - - return orphaned, updated -} - // Hash returns the root hash. -func (t *Tree) Hash() []byte { +func (t *ImmutableTree) Hash() []byte { if t.root == nil { return nil } @@ -108,7 +89,7 @@ func (t *Tree) Hash() []byte { } // hashWithCount returns the root hash and hash count. -func (t *Tree) hashWithCount() ([]byte, int64) { +func (t *ImmutableTree) hashWithCount() ([]byte, int64) { if t.root == nil { return nil, 0 } @@ -117,12 +98,12 @@ func (t *Tree) hashWithCount() ([]byte, int64) { // Get returns the index and value of the specified key if it exists, or nil // and the next index, if it doesn't. -func (t *Tree) Get(key []byte) (index int, value []byte) { +func (t *ImmutableTree) Get(key []byte) (index int, value []byte) { index64, value := t.Get64(key) return int(index64), value } -func (t *Tree) Get64(key []byte) (index int64, value []byte) { +func (t *ImmutableTree) Get64(key []byte) (index int64, value []byte) { if t.root == nil { return 0, nil } @@ -130,45 +111,19 @@ func (t *Tree) Get64(key []byte) (index int64, value []byte) { } // GetByIndex gets the key and value at the specified index. -func (t *Tree) GetByIndex(index int) (key []byte, value []byte) { +func (t *ImmutableTree) GetByIndex(index int) (key []byte, value []byte) { return t.GetByIndex64(int64(index)) } -func (t *Tree) GetByIndex64(index int64) (key []byte, value []byte) { +func (t *ImmutableTree) GetByIndex64(index int64) (key []byte, value []byte) { if t.root == nil { return nil, nil } return t.root.getByIndex(t, index) } -// Remove tries to remove a key from the tree and if removed, returns its -// value, and 'true'. -func (t *Tree) Remove(key []byte) ([]byte, bool) { - value, _, removed := t.remove(key) - return value, removed -} - -// remove tries to remove a key from the tree and if removed, returns its -// value, nodes orphaned and 'true'. -func (t *Tree) remove(key []byte) (value []byte, orphans []*Node, removed bool) { - if t.root == nil { - return nil, nil, false - } - newRootHash, newRoot, _, value, orphaned := t.root.remove(t, key) - if len(orphaned) == 0 { - return nil, nil, false - } - - if newRoot == nil && newRootHash != nil { - t.root = t.ndb.GetNode(newRootHash) - } else { - t.root = newRoot - } - return value, orphaned, true -} - // Iterate iterates over all keys of the tree, in order. -func (t *Tree) Iterate(fn func(key []byte, value []byte) bool) (stopped bool) { +func (t *ImmutableTree) Iterate(fn func(key []byte, value []byte) bool) (stopped bool) { if t.root == nil { return false } @@ -182,7 +137,7 @@ func (t *Tree) Iterate(fn func(key []byte, value []byte) bool) (stopped bool) { // IterateRange makes a callback for all nodes with key between start and end non-inclusive. // If either are nil, then it is open on that side (nil, nil is the same as Iterate) -func (t *Tree) IterateRange(start, end []byte, ascending bool, fn func(key []byte, value []byte) bool) (stopped bool) { +func (t *ImmutableTree) IterateRange(start, end []byte, ascending bool, fn func(key []byte, value []byte) bool) (stopped bool) { if t.root == nil { return false } @@ -196,7 +151,7 @@ func (t *Tree) IterateRange(start, end []byte, ascending bool, fn func(key []byt // IterateRangeInclusive makes a callback for all nodes with key between start and end inclusive. // If either are nil, then it is open on that side (nil, nil is the same as Iterate) -func (t *Tree) IterateRangeInclusive(start, end []byte, ascending bool, fn func(key, value []byte, version int64) bool) (stopped bool) { +func (t *ImmutableTree) IterateRangeInclusive(start, end []byte, ascending bool, fn func(key, value []byte, version int64) bool) (stopped bool) { if t.root == nil { return false } @@ -209,9 +164,9 @@ func (t *Tree) IterateRangeInclusive(start, end []byte, ascending bool, fn func( } // Clone creates a clone of the tree. -// Used internally by VersionedTree. -func (t *Tree) clone() *Tree { - return &Tree{ +// Used internally by MutableTree. +func (t *ImmutableTree) clone() *ImmutableTree { + return &ImmutableTree{ root: t.root, ndb: t.ndb, version: t.version, @@ -219,7 +174,7 @@ func (t *Tree) clone() *Tree { } // nodeSize is like Size, but includes inner nodes too. -func (t *Tree) nodeSize() int { +func (t *ImmutableTree) nodeSize() int { size := 0 t.root.traverse(t, true, func(n *Node) bool { size++ diff --git a/mutable_tree.go b/mutable_tree.go new file mode 100644 index 000000000..8ad073345 --- /dev/null +++ b/mutable_tree.go @@ -0,0 +1,469 @@ +package iavl + +import ( + "bytes" + "fmt" + + cmn "github.com/tendermint/tendermint/libs/common" + dbm "github.com/tendermint/tendermint/libs/db" +) + +// ErrVersionDoesNotExist is returned if a requested version does not exist. +var ErrVersionDoesNotExist = fmt.Errorf("version does not exist") + +// MutableTree is a persistent tree which keeps track of versions. +type MutableTree struct { + *ImmutableTree // The current, working tree. + lastSaved *ImmutableTree // The most recently saved tree. + orphans map[string]int64 // Nodes removed by changes to working tree. + versions map[int64]bool // The previous, saved versions of the tree. + ndb *nodeDB +} + +// NewMutableTree returns a new tree with the specified cache size and datastore. +func NewMutableTree(db dbm.DB, cacheSize int) *MutableTree { + ndb := newNodeDB(db, cacheSize) + head := &ImmutableTree{ndb: ndb} + + return &MutableTree{ + ImmutableTree: head, + lastSaved: head.clone(), + orphans: map[string]int64{}, + versions: map[int64]bool{}, + ndb: ndb, + } +} + +// IsEmpty returns whether or not the tree has any keys. Only trees that are +// not empty can be saved. +func (tree *MutableTree) IsEmpty() bool { + return tree.ImmutableTree.Size() == 0 +} + +// VersionExists returns whether or not a version exists. +func (tree *MutableTree) VersionExists(version int64) bool { + return tree.versions[version] +} + +// Hash returns the hash of the latest saved version of the tree, as returned +// by SaveVersion. If no versions have been saved, Hash returns nil. +func (tree *MutableTree) Hash() []byte { + if tree.version > 0 { + return tree.lastSaved.Hash() + } + return nil +} + +// WorkingHash returns the hash of the current working tree. +func (tree *MutableTree) WorkingHash() []byte { + return tree.ImmutableTree.Hash() +} + +// String returns a string representation of the tree. +func (tree *MutableTree) String() string { + return tree.ndb.String() +} + +// Set sets a key in the working tree. Nil values are not supported. +func (tree *MutableTree) Set(key, value []byte) bool { + orphaned, updated := tree.set(key, value) + tree.addOrphans(orphaned) + return updated +} + +func (tree *MutableTree) set(key []byte, value []byte) (orphaned []*Node, updated bool) { + if value == nil { + panic(fmt.Sprintf("Attempt to store nil value at key '%s'", key)) + } + if tree.ImmutableTree.root == nil { + tree.ImmutableTree.root = NewNode(key, value, tree.version+1) + return nil, false + } + tree.ImmutableTree.root, updated, orphaned = tree.recursiveSet(tree.ImmutableTree.root, key, value) + + return orphaned, updated +} + +func (tree *MutableTree) recursiveSet(node *Node, key []byte, value []byte) ( + newSelf *Node, updated bool, orphaned []*Node, +) { + version := tree.version + 1 + + if node.isLeaf() { + switch bytes.Compare(key, node.key) { + case -1: + return &Node{ + key: node.key, + height: 1, + size: 2, + leftNode: NewNode(key, value, version), + rightNode: node, + version: version, + }, false, []*Node{} + case 1: + return &Node{ + key: key, + height: 1, + size: 2, + leftNode: node, + rightNode: NewNode(key, value, version), + version: version, + }, false, []*Node{} + default: + return NewNode(key, value, version), true, []*Node{node} + } + } else { + orphaned = append(orphaned, node) + node = node.clone(version) + + if bytes.Compare(key, node.key) < 0 { + var leftOrphaned []*Node + node.leftNode, updated, leftOrphaned = tree.recursiveSet(node.getLeftNode(tree.ImmutableTree), key, value) + node.leftHash = nil // leftHash is yet unknown + orphaned = append(orphaned, leftOrphaned...) + } else { + var rightOrphaned []*Node + node.rightNode, updated, rightOrphaned = tree.recursiveSet(node.getRightNode(tree.ImmutableTree), key, value) + node.rightHash = nil // rightHash is yet unknown + orphaned = append(orphaned, rightOrphaned...) + } + + if updated { + return node, updated, orphaned + } + node.calcHeightAndSize(tree.ImmutableTree) + newNode, balanceOrphaned := tree.balance(node) + return newNode, updated, append(orphaned, balanceOrphaned...) + } +} + +// Remove removes a key from the working tree. +func (tree *MutableTree) Remove(key []byte) ([]byte, bool) { + val, orphaned, removed := tree.remove(key) + tree.addOrphans(orphaned) + return val, removed +} + +// remove tries to remove a key from the tree and if removed, returns its +// value, nodes orphaned and 'true'. +func (tree *MutableTree) remove(key []byte) (value []byte, orphans []*Node, removed bool) { + if tree.root == nil { + return nil, nil, false + } + newRootHash, newRoot, _, value, orphaned := tree.recursiveRemove(tree.root, key) + if len(orphaned) == 0 { + return nil, nil, false + } + + if newRoot == nil && newRootHash != nil { + tree.root = tree.ndb.GetNode(newRootHash) + } else { + tree.root = newRoot + } + return value, orphaned, true +} + +// removes the node corresponding to the passed key and balances the tree. +// It returns: +// - the hash of the new node (or nil if the node is the one removed) +// - the node that replaces the orig. node after remove +// - new leftmost leaf key for tree after successfully removing 'key' if changed. +// - the removed value +// - the orphaned nodes. +func (tree *MutableTree) recursiveRemove(node *Node, key []byte) ([]byte, *Node, []byte, []byte, []*Node) { + version := tree.version + 1 + + if node.isLeaf() { + if bytes.Equal(key, node.key) { + return nil, nil, nil, node.value, []*Node{node} + } + return node.hash, node, nil, nil, nil + } + + // node.key < key; we go to the left to find the key: + if bytes.Compare(key, node.key) < 0 { + newLeftHash, newLeftNode, newKey, value, orphaned := tree.recursiveRemove(node.getLeftNode(tree.ImmutableTree), key) + + if len(orphaned) == 0 { + return node.hash, node, nil, value, orphaned + } else if newLeftHash == nil && newLeftNode == nil { // left node held value, was removed + return node.rightHash, node.rightNode, node.key, value, orphaned + } + orphaned = append(orphaned, node) + + newNode := node.clone(version) + newNode.leftHash, newNode.leftNode = newLeftHash, newLeftNode + newNode.calcHeightAndSize(tree.ImmutableTree) + newNode, balanceOrphaned := tree.balance(newNode) + + return newNode.hash, newNode, newKey, value, append(orphaned, balanceOrphaned...) + } + // node.key >= key; either found or look to the right: + newRightHash, newRightNode, newKey, value, orphaned := tree.recursiveRemove(node.getRightNode(tree.ImmutableTree), key) + + if len(orphaned) == 0 { + return node.hash, node, nil, value, orphaned + } else if newRightHash == nil && newRightNode == nil { // right node held value, was removed + return node.leftHash, node.leftNode, nil, value, orphaned + } + orphaned = append(orphaned, node) + + newNode := node.clone(version) + newNode.rightHash, newNode.rightNode = newRightHash, newRightNode + if newKey != nil { + newNode.key = newKey + } + newNode.calcHeightAndSize(tree.ImmutableTree) + newNode, balanceOrphaned := tree.balance(newNode) + + return newNode.hash, newNode, nil, value, append(orphaned, balanceOrphaned...) +} + +// Load the latest versioned tree from disk. +func (tree *MutableTree) Load() (int64, error) { + return tree.LoadVersion(int64(0)) +} + +// Returns the version number of the latest version found +func (tree *MutableTree) LoadVersion(targetVersion int64) (int64, error) { + roots, err := tree.ndb.getRoots() + if err != nil { + return 0, err + } + if len(roots) == 0 { + return 0, nil + } + latestVersion := int64(0) + var latestRoot []byte + for version, r := range roots { + tree.versions[version] = true + if version > latestVersion && + (targetVersion == 0 || version <= targetVersion) { + latestVersion = version + latestRoot = r + } + } + + if !(targetVersion == 0 || latestVersion == targetVersion) { + return latestVersion, fmt.Errorf("wanted to load target %v but only found up to %v", + targetVersion, latestVersion) + } + + t := &ImmutableTree{ + ndb: tree.ndb, + version: latestVersion, + } + if len(latestRoot) != 0 { + t.root = tree.ndb.GetNode(latestRoot) + } + + tree.orphans = map[string]int64{} + tree.ImmutableTree = t + tree.lastSaved = t.clone() + return latestVersion, nil +} + +// GetImmutable loads an ImmutableTree at a given version for querying +func (tree *MutableTree) GetImmutable(version int64) (*ImmutableTree, error) { + rootHash := tree.ndb.getRoot(version) + if rootHash == nil { + return nil, ErrVersionDoesNotExist + } else if len(rootHash) == 0 { + return &ImmutableTree{ + ndb: tree.ndb, + version: version, + }, nil + } + return &ImmutableTree{ + root: tree.ndb.GetNode(rootHash), + ndb: tree.ndb, + version: version, + }, nil +} + +// Rollback resets the working tree to the latest saved version, discarding +// any unsaved modifications. +func (tree *MutableTree) Rollback() { + if tree.version > 0 { + tree.ImmutableTree = tree.lastSaved.clone() + } else { + tree.ImmutableTree = &ImmutableTree{ndb: tree.ndb, version: 0} + } + tree.orphans = map[string]int64{} +} + +// GetVersioned gets the value at the specified key and version. +func (tree *MutableTree) GetVersioned(key []byte, version int64) ( + index int, value []byte, +) { + if tree.versions[version] { + t, err := tree.GetImmutable(version) + if err != nil { + return -1, nil + } + return t.Get(key) + } + return -1, nil +} + +// SaveVersion saves a new tree version to disk, based on the current state of +// the tree. Returns the hash and new version number. +func (tree *MutableTree) SaveVersion() ([]byte, int64, error) { + version := tree.version + 1 + + if tree.versions[version] { + //version already exists, throw an error if attempting to overwrite + // Same hash means idempotent. Return success. + existingHash := tree.ndb.getRoot(version) + var newHash = tree.WorkingHash() + if bytes.Equal(existingHash, newHash) { + tree.version = version + tree.ImmutableTree = tree.ImmutableTree.clone() + tree.lastSaved = tree.ImmutableTree.clone() + tree.orphans = map[string]int64{} + return existingHash, version, nil + } + return nil, version, fmt.Errorf("version %d was already saved to different hash %X (existing hash %X)", + version, newHash, existingHash) + } + + if tree.root == nil { + // There can still be orphans, for example if the root is the node being + // removed. + debug("SAVE EMPTY TREE %v\n", version) + tree.ndb.SaveOrphans(version, tree.orphans) + tree.ndb.SaveEmptyRoot(version) + } else { + debug("SAVE TREE %v\n", version) + // Save the current tree. + tree.ndb.SaveBranch(tree.root) + tree.ndb.SaveOrphans(version, tree.orphans) + tree.ndb.SaveRoot(tree.root, version) + } + tree.ndb.Commit() + tree.version = version + tree.versions[version] = true + + // Set new working tree. + tree.ImmutableTree = tree.ImmutableTree.clone() + tree.lastSaved = tree.ImmutableTree.clone() + tree.orphans = map[string]int64{} + + return tree.Hash(), version, nil +} + +// DeleteVersion deletes a tree version from disk. The version can then no +// longer be accessed. +func (tree *MutableTree) DeleteVersion(version int64) error { + if version == 0 { + return cmn.NewError("version must be greater than 0") + } + if version == tree.version { + return cmn.NewError("cannot delete latest saved version (%d)", version) + } + if _, ok := tree.versions[version]; !ok { + return cmn.ErrorWrap(ErrVersionDoesNotExist, "") + } + + tree.ndb.DeleteVersion(version) + tree.ndb.Commit() + + delete(tree.versions, version) + + return nil +} + +// Rotate right and return the new node and orphan. +func (tree *MutableTree) rotateRight(node *Node) (*Node, *Node) { + version := tree.version + 1 + + // TODO: optimize balance & rotate. + node = node.clone(version) + orphaned := node.getLeftNode(tree.ImmutableTree) + newNode := orphaned.clone(version) + + newNoderHash, newNoderCached := newNode.rightHash, newNode.rightNode + newNode.rightHash, newNode.rightNode = node.hash, node + node.leftHash, node.leftNode = newNoderHash, newNoderCached + + node.calcHeightAndSize(tree.ImmutableTree) + newNode.calcHeightAndSize(tree.ImmutableTree) + + return newNode, orphaned +} + +// Rotate left and return the new node and orphan. +func (tree *MutableTree) rotateLeft(node *Node) (*Node, *Node) { + version := tree.version + 1 + + // TODO: optimize balance & rotate. + node = node.clone(version) + orphaned := node.getRightNode(tree.ImmutableTree) + newNode := orphaned.clone(version) + + newNodelHash, newNodelCached := newNode.leftHash, newNode.leftNode + newNode.leftHash, newNode.leftNode = node.hash, node + node.rightHash, node.rightNode = newNodelHash, newNodelCached + + node.calcHeightAndSize(tree.ImmutableTree) + newNode.calcHeightAndSize(tree.ImmutableTree) + + return newNode, orphaned +} + +// NOTE: assumes that node can be modified +// TODO: optimize balance & rotate +func (tree *MutableTree) balance(node *Node) (newSelf *Node, orphaned []*Node) { + if node.persisted { + panic("Unexpected balance() call on persisted node") + } + balance := node.calcBalance(tree.ImmutableTree) + + if balance > 1 { + if node.getLeftNode(tree.ImmutableTree).calcBalance(tree.ImmutableTree) >= 0 { + // Left Left Case + newNode, orphaned := tree.rotateRight(node) + return newNode, []*Node{orphaned} + } + // Left Right Case + var leftOrphaned *Node + + left := node.getLeftNode(tree.ImmutableTree) + node.leftHash = nil + node.leftNode, leftOrphaned = tree.rotateLeft(left) + newNode, rightOrphaned := tree.rotateRight(node) + + return newNode, []*Node{left, leftOrphaned, rightOrphaned} + } + if balance < -1 { + if node.getRightNode(tree.ImmutableTree).calcBalance(tree.ImmutableTree) <= 0 { + // Right Right Case + newNode, orphaned := tree.rotateLeft(node) + return newNode, []*Node{orphaned} + } + // Right Left Case + var rightOrphaned *Node + + right := node.getRightNode(tree.ImmutableTree) + node.rightHash = nil + node.rightNode, rightOrphaned = tree.rotateRight(right) + newNode, leftOrphaned := tree.rotateLeft(node) + + return newNode, []*Node{right, leftOrphaned, rightOrphaned} + } + // Nothing changed + return node, []*Node{} +} + +func (tree *MutableTree) addOrphans(orphans []*Node) { + for _, node := range orphans { + if !node.persisted { + // We don't need to orphan nodes that were never persisted. + continue + } + if len(node.hash) == 0 { + panic("Expected to find node hash, but was empty") + } + tree.orphans[string(node.hash)] = node.version + } +} diff --git a/node.go b/node.go index 307412c33..863751b6a 100644 --- a/node.go +++ b/node.go @@ -140,7 +140,7 @@ func (node *Node) isLeaf() bool { } // Check if the node has a descendant with the given key. -func (node *Node) has(t *Tree, key []byte) (has bool) { +func (node *Node) has(t *ImmutableTree, key []byte) (has bool) { if bytes.Equal(node.key, key) { return true } @@ -154,7 +154,7 @@ func (node *Node) has(t *Tree, key []byte) (has bool) { } // Get a key under the node. -func (node *Node) get(t *Tree, key []byte) (index int64, value []byte) { +func (node *Node) get(t *ImmutableTree, key []byte) (index int64, value []byte) { if node.isLeaf() { switch bytes.Compare(node.key, key) { case -1: @@ -175,7 +175,7 @@ func (node *Node) get(t *Tree, key []byte) (index int64, value []byte) { return index, value } -func (node *Node) getByIndex(t *Tree, index int64) (key []byte, value []byte) { +func (node *Node) getByIndex(t *ImmutableTree, index int64) (key []byte, value []byte) { if node.isLeaf() { if index == 0 { return node.key, node.value @@ -341,233 +341,42 @@ func (node *Node) writeBytes(w io.Writer) cmn.Error { return nil } -func (node *Node) set(t *Tree, key []byte, value []byte) ( - newSelf *Node, updated bool, orphaned []*Node, -) { - version := t.version + 1 - - if node.isLeaf() { - switch bytes.Compare(key, node.key) { - case -1: - return &Node{ - key: node.key, - height: 1, - size: 2, - leftNode: NewNode(key, value, version), - rightNode: node, - version: version, - }, false, []*Node{} - case 1: - return &Node{ - key: key, - height: 1, - size: 2, - leftNode: node, - rightNode: NewNode(key, value, version), - version: version, - }, false, []*Node{} - default: - return NewNode(key, value, version), true, []*Node{node} - } - } else { - orphaned = append(orphaned, node) - node = node.clone(version) - - if bytes.Compare(key, node.key) < 0 { - var leftOrphaned []*Node - node.leftNode, updated, leftOrphaned = node.getLeftNode(t).set(t, key, value) - node.leftHash = nil // leftHash is yet unknown - orphaned = append(orphaned, leftOrphaned...) - } else { - var rightOrphaned []*Node - node.rightNode, updated, rightOrphaned = node.getRightNode(t).set(t, key, value) - node.rightHash = nil // rightHash is yet unknown - orphaned = append(orphaned, rightOrphaned...) - } - - if updated { - return node, updated, orphaned - } - node.calcHeightAndSize(t) - newNode, balanceOrphaned := node.balance(t) - return newNode, updated, append(orphaned, balanceOrphaned...) - } -} - -// removes the node corresponding to the passed key and balances the tree. -// It returns: -// - the hash of the new node (or nil if the node is the one removed) -// - the node that replaces the orig. node after remove -// - new leftmost leaf key for tree after successfully removing 'key' if changed. -// - the removed value -// - the orphaned nodes. -func (node *Node) remove(t *Tree, key []byte) ([]byte, *Node, []byte, []byte, []*Node) { - version := t.version + 1 - - if node.isLeaf() { - if bytes.Equal(key, node.key) { - return nil, nil, nil, node.value, []*Node{node} - } - return node.hash, node, nil, nil, nil - } - - // node.key < key; we go to the left to find the key: - if bytes.Compare(key, node.key) < 0 { - newLeftHash, newLeftNode, newKey, value, orphaned := node.getLeftNode(t).remove(t, key) - - if len(orphaned) == 0 { - return node.hash, node, nil, value, orphaned - } else if newLeftHash == nil && newLeftNode == nil { // left node held value, was removed - return node.rightHash, node.rightNode, node.key, value, orphaned - } - orphaned = append(orphaned, node) - - newNode := node.clone(version) - newNode.leftHash, newNode.leftNode = newLeftHash, newLeftNode - newNode.calcHeightAndSize(t) - newNode, balanceOrphaned := newNode.balance(t) - - return newNode.hash, newNode, newKey, value, append(orphaned, balanceOrphaned...) - } - // node.key >= key; either found or look to the right: - newRightHash, newRightNode, newKey, value, orphaned := node.getRightNode(t).remove(t, key) - - if len(orphaned) == 0 { - return node.hash, node, nil, value, orphaned - } else if newRightHash == nil && newRightNode == nil { // right node held value, was removed - return node.leftHash, node.leftNode, nil, value, orphaned - } - orphaned = append(orphaned, node) - - newNode := node.clone(version) - newNode.rightHash, newNode.rightNode = newRightHash, newRightNode - if newKey != nil { - newNode.key = newKey - } - newNode.calcHeightAndSize(t) - newNode, balanceOrphaned := newNode.balance(t) - - return newNode.hash, newNode, nil, value, append(orphaned, balanceOrphaned...) -} - -func (node *Node) getLeftNode(t *Tree) *Node { +func (node *Node) getLeftNode(t *ImmutableTree) *Node { if node.leftNode != nil { return node.leftNode } return t.ndb.GetNode(node.leftHash) } -func (node *Node) getRightNode(t *Tree) *Node { +func (node *Node) getRightNode(t *ImmutableTree) *Node { if node.rightNode != nil { return node.rightNode } return t.ndb.GetNode(node.rightHash) } -// Rotate right and return the new node and orphan. -func (node *Node) rotateRight(t *Tree) (newNode *Node, orphan *Node) { - version := t.version + 1 - - // TODO: optimize balance & rotate. - node = node.clone(version) - l := node.getLeftNode(t) - _l := l.clone(version) - - _lrHash, _lrCached := _l.rightHash, _l.rightNode - _l.rightHash, _l.rightNode = node.hash, node - node.leftHash, node.leftNode = _lrHash, _lrCached - - node.calcHeightAndSize(t) - _l.calcHeightAndSize(t) - - return _l, l -} - -// Rotate left and return the new node and orphan. -func (node *Node) rotateLeft(t *Tree) (newNode *Node, orphan *Node) { - version := t.version + 1 - - // TODO: optimize balance & rotate. - node = node.clone(version) - r := node.getRightNode(t) - _r := r.clone(version) - - _rlHash, _rlCached := _r.leftHash, _r.leftNode - _r.leftHash, _r.leftNode = node.hash, node - node.rightHash, node.rightNode = _rlHash, _rlCached - - node.calcHeightAndSize(t) - _r.calcHeightAndSize(t) - - return _r, r -} - // NOTE: mutates height and size -func (node *Node) calcHeightAndSize(t *Tree) { +func (node *Node) calcHeightAndSize(t *ImmutableTree) { node.height = maxInt8(node.getLeftNode(t).height, node.getRightNode(t).height) + 1 node.size = node.getLeftNode(t).size + node.getRightNode(t).size } -func (node *Node) calcBalance(t *Tree) int { +func (node *Node) calcBalance(t *ImmutableTree) int { return int(node.getLeftNode(t).height) - int(node.getRightNode(t).height) } -// NOTE: assumes that node can be modified -// TODO: optimize balance & rotate -func (node *Node) balance(t *Tree) (newSelf *Node, orphaned []*Node) { - if node.persisted { - panic("Unexpected balance() call on persisted node") - } - balance := node.calcBalance(t) - - if balance > 1 { - if node.getLeftNode(t).calcBalance(t) >= 0 { - // Left Left Case - newNode, orphaned := node.rotateRight(t) - return newNode, []*Node{orphaned} - } - // Left Right Case - var leftOrphaned *Node - - left := node.getLeftNode(t) - node.leftHash = nil - node.leftNode, leftOrphaned = left.rotateLeft(t) - newNode, rightOrphaned := node.rotateRight(t) - - return newNode, []*Node{left, leftOrphaned, rightOrphaned} - } - if balance < -1 { - if node.getRightNode(t).calcBalance(t) <= 0 { - // Right Right Case - newNode, orphaned := node.rotateLeft(t) - return newNode, []*Node{orphaned} - } - // Right Left Case - var rightOrphaned *Node - - right := node.getRightNode(t) - node.rightHash = nil - node.rightNode, rightOrphaned = right.rotateRight(t) - newNode, leftOrphaned := node.rotateLeft(t) - - return newNode, []*Node{right, leftOrphaned, rightOrphaned} - } - // Nothing changed - return node, []*Node{} -} - // traverse is a wrapper over traverseInRange when we want the whole tree -func (node *Node) traverse(t *Tree, ascending bool, cb func(*Node) bool) bool { +func (node *Node) traverse(t *ImmutableTree, ascending bool, cb func(*Node) bool) bool { return node.traverseInRange(t, nil, nil, ascending, false, 0, func(node *Node, depth uint8) bool { return cb(node) }) } -func (node *Node) traverseWithDepth(t *Tree, ascending bool, cb func(*Node, uint8) bool) bool { +func (node *Node) traverseWithDepth(t *ImmutableTree, ascending bool, cb func(*Node, uint8) bool) bool { return node.traverseInRange(t, nil, nil, ascending, false, 0, cb) } -func (node *Node) traverseInRange(t *Tree, start, end []byte, ascending bool, inclusive bool, depth uint8, cb func(*Node, uint8) bool) bool { +func (node *Node) traverseInRange(t *ImmutableTree, start, end []byte, ascending bool, inclusive bool, depth uint8, cb func(*Node, uint8) bool) bool { afterStart := start == nil || bytes.Compare(start, node.key) < 0 startOrAfter := start == nil || bytes.Compare(start, node.key) <= 0 beforeEnd := end == nil || bytes.Compare(node.key, end) < 0 @@ -615,7 +424,7 @@ func (node *Node) traverseInRange(t *Tree, start, end []byte, ascending bool, in } // Only used in testing... -func (node *Node) lmd(t *Tree) *Node { +func (node *Node) lmd(t *ImmutableTree) *Node { if node.isLeaf() { return node } diff --git a/nodedb.go b/nodedb.go index d0d3df19c..a933b2662 100644 --- a/nodedb.go +++ b/nodedb.go @@ -332,6 +332,10 @@ func (ndb *nodeDB) Commit() { ndb.batch = ndb.db.NewBatch() } +func (ndb *nodeDB) getRoot(version int64) []byte { + return ndb.db.Get(ndb.rootKey(version)) +} + func (ndb *nodeDB) getRoots() (map[int64][]byte, error) { roots := map[int64][]byte{} diff --git a/orphaning_tree.go b/orphaning_tree.go deleted file mode 100644 index fb7493f28..000000000 --- a/orphaning_tree.go +++ /dev/null @@ -1,74 +0,0 @@ -package iavl - -import ( - "fmt" -) - -// orphaningTree is a tree which keeps track of orphaned nodes. -type orphaningTree struct { - *Tree - - // A map of orphan hash to orphan version. - // The version stored here is the one at which the orphan's lifetime - // begins. - orphans map[string]int64 -} - -// newOrphaningTree creates a new orphaning tree from the given *Tree. -func newOrphaningTree(t *Tree) *orphaningTree { - return &orphaningTree{ - Tree: t, - orphans: map[string]int64{}, - } -} - -// Set a key on the underlying tree while storing the orphaned nodes. -func (tree *orphaningTree) Set(key, value []byte) bool { - orphaned, updated := tree.Tree.set(key, value) - tree.addOrphans(orphaned) - return updated -} - -// Remove a key from the underlying tree while storing the orphaned nodes. -func (tree *orphaningTree) Remove(key []byte) ([]byte, bool) { - val, orphaned, removed := tree.Tree.remove(key) - tree.addOrphans(orphaned) - return val, removed -} - -// SaveAs saves the underlying Tree and assigns it a new version. -// Saves orphans too. -func (tree *orphaningTree) SaveAs(version int64) { - if version != tree.version+1 { - panic(fmt.Sprintf("Expected to save version %d but tried to save %d", tree.version+1, version)) - } - if tree.root == nil { - // There can still be orphans, for example if the root is the node being - // removed. - debug("SAVE EMPTY TREE %v\n", version) - tree.ndb.SaveOrphans(version, tree.orphans) - tree.ndb.SaveEmptyRoot(version) - } else { - debug("SAVE TREE %v\n", version) - // Save the current tree. - tree.ndb.SaveBranch(tree.root) - tree.ndb.SaveOrphans(version, tree.orphans) - tree.ndb.SaveRoot(tree.root, version) - } - tree.ndb.Commit() - tree.version = version -} - -// Add orphans to the orphan list. Doesn't write to disk. -func (tree *orphaningTree) addOrphans(orphans []*Node) { - for _, node := range orphans { - if !node.persisted { - // We don't need to orphan nodes that were never persisted. - continue - } - if len(node.hash) == 0 { - panic("Expected to find node hash, but was empty") - } - tree.orphans[string(node.hash)] = node.version - } -} diff --git a/proof.go b/proof.go index a87877048..b68293ccc 100644 --- a/proof.go +++ b/proof.go @@ -142,7 +142,7 @@ func (pln proofLeafNode) Hash() []byte { // If the key does not exist, returns the path to the next leaf left of key (w/ // path), except when key is less than the least item, in which case it returns // a path to the least item. -func (node *Node) PathToLeaf(t *Tree, key []byte) (PathToLeaf, *Node, error) { +func (node *Node) PathToLeaf(t *ImmutableTree, key []byte) (PathToLeaf, *Node, error) { path := new(PathToLeaf) val, err := node.pathToLeaf(t, key, path) return *path, val, err @@ -151,7 +151,7 @@ func (node *Node) PathToLeaf(t *Tree, key []byte) (PathToLeaf, *Node, error) { // pathToLeaf is a helper which recursively constructs the PathToLeaf. // As an optimization the already constructed path is passed in as an argument // and is shared among recursive calls. -func (node *Node) pathToLeaf(t *Tree, key []byte, path *PathToLeaf) (*Node, error) { +func (node *Node) pathToLeaf(t *ImmutableTree, key []byte, path *PathToLeaf) (*Node, error) { if node.height == 0 { if bytes.Equal(node.key, key) { return node, nil diff --git a/proof_range.go b/proof_range.go index cc12618f9..47691c527 100644 --- a/proof_range.go +++ b/proof_range.go @@ -26,7 +26,7 @@ type RangeProof struct { // Keys returns all the keys in the RangeProof. NOTE: The keys here may // include more keys than provided by tree.GetRangeWithProof or -// VersionedTree.GetVersionedRangeWithProof. The keys returned there are only +// MutableTree.GetVersionedRangeWithProof. The keys returned there are only // in the provided [startKey,endKey){limit} range. The keys returned here may // include extra keys, such as: // - the key before startKey if startKey is provided and doesn't exist; @@ -308,7 +308,7 @@ func (proof *RangeProof) _computeRootHash() (rootHash []byte, treeEnd bool, err // If keyEnd-1 exists, no later leaves will be included. // If keyStart >= keyEnd and both not nil, panics. // Limit is never exceeded. -func (t *Tree) getRangeProof(keyStart, keyEnd []byte, limit int) (*RangeProof, [][]byte, [][]byte, error) { +func (t *ImmutableTree) getRangeProof(keyStart, keyEnd []byte, limit int) (*RangeProof, [][]byte, [][]byte, error) { if keyStart != nil && keyEnd != nil && bytes.Compare(keyStart, keyEnd) >= 0 { panic("if keyStart and keyEnd are present, need keyStart < keyEnd.") } @@ -451,7 +451,7 @@ func (t *Tree) getRangeProof(keyStart, keyEnd []byte, limit int) (*RangeProof, [ // GetWithProof gets the value under the key if it exists, or returns nil. // A proof of existence or absence is returned alongside the value. -func (t *Tree) GetWithProof(key []byte) (value []byte, proof *RangeProof, err error) { +func (t *ImmutableTree) GetWithProof(key []byte) (value []byte, proof *RangeProof, err error) { proof, _, values, err := t.getRangeProof(key, cpIncr(key), 2) if err == nil { if len(values) > 0 { @@ -466,15 +466,19 @@ func (t *Tree) GetWithProof(key []byte) (value []byte, proof *RangeProof, err er } // GetRangeWithProof gets key/value pairs within the specified range and limit. -func (t *Tree) GetRangeWithProof(startKey []byte, endKey []byte, limit int) (keys, values [][]byte, proof *RangeProof, err error) { +func (t *ImmutableTree) GetRangeWithProof(startKey []byte, endKey []byte, limit int) (keys, values [][]byte, proof *RangeProof, err error) { proof, keys, values, err = t.getRangeProof(startKey, endKey, limit) return } // GetVersionedWithProof gets the value under the key at the specified version // if it exists, or returns nil. -func (tree *VersionedTree) GetVersionedWithProof(key []byte, version int64) ([]byte, *RangeProof, error) { - if t, ok := tree.versions[version]; ok { +func (tree *MutableTree) GetVersionedWithProof(key []byte, version int64) ([]byte, *RangeProof, error) { + if tree.versions[version] { + t, err := tree.GetImmutable(version) + if err != nil { + return nil, nil, err + } return t.GetWithProof(key) } return nil, nil, cmn.ErrorWrap(ErrVersionDoesNotExist, "") @@ -482,10 +486,14 @@ func (tree *VersionedTree) GetVersionedWithProof(key []byte, version int64) ([]b // GetVersionedRangeWithProof gets key/value pairs within the specified range // and limit. -func (tree *VersionedTree) GetVersionedRangeWithProof(startKey, endKey []byte, limit int, version int64) ( +func (tree *MutableTree) GetVersionedRangeWithProof(startKey, endKey []byte, limit int, version int64) ( keys, values [][]byte, proof *RangeProof, err error) { - if t, ok := tree.versions[version]; ok { + if tree.versions[version] { + t, err := tree.GetImmutable(version) + if err != nil { + return nil, nil, nil, err + } return t.GetRangeWithProof(startKey, endKey, limit) } return nil, nil, nil, cmn.ErrorWrap(ErrVersionDoesNotExist, "") diff --git a/proof_test.go b/proof_test.go index 53cc5c977..b408a0d9f 100644 --- a/proof_test.go +++ b/proof_test.go @@ -8,17 +8,18 @@ import ( "github.com/stretchr/testify/require" "github.com/tendermint/go-amino" + "github.com/tendermint/tendermint/libs/db" "github.com/tendermint/tendermint/libs/test" ) func TestTreeGetWithProof(t *testing.T) { - tree := NewTree(nil, 0) + tree := NewMutableTree(db.NewMemDB(), 0) require := require.New(t) for _, ikey := range []byte{0x11, 0x32, 0x50, 0x72, 0x99} { key := []byte{ikey} tree.Set(key, []byte(rand.Str(8))) } - root := tree.Hash() + root := tree.WorkingHash() key := []byte{0x32} val, proof, err := tree.GetWithProof(key) @@ -46,8 +47,8 @@ func TestTreeGetWithProof(t *testing.T) { } func TestTreeKeyExistsProof(t *testing.T) { - tree := NewTree(nil, 0) - root := tree.Hash() + tree := NewMutableTree(db.NewMemDB(), 0) + root := tree.WorkingHash() // should get false for proof with nil root proof, _, _, err := tree.getRangeProof([]byte("foo"), nil, 1) @@ -63,7 +64,7 @@ func TestTreeKeyExistsProof(t *testing.T) { allkeys[i] = []byte(key) } sortByteSlices(allkeys) // Sort all keys - root = tree.Hash() + root = tree.WorkingHash() // query random key fails proof, _, _, err = tree.getRangeProof([]byte("foo"), nil, 2) @@ -109,14 +110,14 @@ func TestTreeKeyExistsProof(t *testing.T) { } func TestTreeKeyInRangeProofs(t *testing.T) { - tree := NewTree(nil, 0) + tree := NewMutableTree(db.NewMemDB(), 0) require := require.New(t) keys := []byte{0x0a, 0x11, 0x2e, 0x32, 0x50, 0x72, 0x99, 0xa1, 0xe4, 0xf7} // 10 total. for _, ikey := range keys { key := []byte{ikey} tree.Set(key, key) } - root := tree.Hash() + root := tree.WorkingHash() // For spacing: T := 10 diff --git a/testutils_test.go b/testutils_test.go index 1e727fb52..1a76d60b9 100644 --- a/testutils_test.go +++ b/testutils_test.go @@ -53,9 +53,9 @@ func N(l, r interface{}) *Node { } // Setup a deep node -func T(n *Node) *Tree { +func T(n *Node) *MutableTree { d := db.NewDB("test", db.MemDBBackend, "") - t := NewTree(d, 0) + t := NewMutableTree(d, 0) n.hashWithCount() t.root = n @@ -114,7 +114,7 @@ func benchmarkImmutableAvlTreeWithDB(b *testing.B, db db.DB) { b.StopTimer() - t := NewVersionedTree(db, 100000) + t := NewMutableTree(db, 100000) for i := 0; i < 1000000; i++ { t.Set(i2b(int(cmn.RandInt32())), nil) if i > 990000 && i%1000 == 999 { diff --git a/tree_dotgraph.go b/tree_dotgraph.go index 14294851a..c6f50374d 100644 --- a/tree_dotgraph.go +++ b/tree_dotgraph.go @@ -41,7 +41,7 @@ var defaultGraphNodeAttrs = map[string]string{ "shape": "circle", } -func WriteDOTGraph(w io.Writer, tree *Tree, paths []PathToLeaf) { +func WriteDOTGraph(w io.Writer, tree *ImmutableTree, paths []PathToLeaf) { ctx := &graphContext{} tree.root.hashWithCount() diff --git a/tree_dotgraph_test.go b/tree_dotgraph_test.go index 02b245c5d..cec1475bd 100644 --- a/tree_dotgraph_test.go +++ b/tree_dotgraph_test.go @@ -3,15 +3,17 @@ package iavl import ( "io/ioutil" "testing" + + "github.com/tendermint/tendermint/libs/db" ) func TestWriteDOTGraph(t *testing.T) { - tree := NewTree(nil, 0) + tree := NewMutableTree(db.NewMemDB(), 0) for _, ikey := range []byte{ 0x0a, 0x11, 0x2e, 0x32, 0x50, 0x72, 0x99, 0xa1, 0xe4, 0xf7, } { key := []byte{ikey} tree.Set(key, key) } - WriteDOTGraph(ioutil.Discard, tree, []PathToLeaf{}) + WriteDOTGraph(ioutil.Discard, tree.ImmutableTree, []PathToLeaf{}) } diff --git a/tree_fuzz_test.go b/tree_fuzz_test.go index 59d693153..d5e0bcaf9 100644 --- a/tree_fuzz_test.go +++ b/tree_fuzz_test.go @@ -16,7 +16,7 @@ type program struct { instructions []instruction } -func (p *program) Execute(tree *VersionedTree) (err error) { +func (p *program) Execute(tree *MutableTree) (err error) { var errLine int defer func() { @@ -55,7 +55,7 @@ type instruction struct { version int64 } -func (i instruction) Execute(tree *VersionedTree) { +func (i instruction) Execute(tree *MutableTree) { switch i.op { case "SET": tree.Set(i.k, i.v) @@ -103,14 +103,14 @@ func genRandomProgram(size int) *program { } // Generate many programs and run them. -func TestVersionedTreeFuzz(t *testing.T) { +func TestMutableTreeFuzz(t *testing.T) { maxIterations := testFuzzIterations progsPerIteration := 100000 iterations := 0 for size := 5; iterations < maxIterations; size++ { for i := 0; i < progsPerIteration/size; i++ { - tree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(db.NewMemDB(), 0) program := genRandomProgram(size) err := program.Execute(tree) if err != nil { diff --git a/tree_test.go b/tree_test.go index d4d0ff0d3..c12eb7116 100644 --- a/tree_test.go +++ b/tree_test.go @@ -3,6 +3,7 @@ package iavl import ( "bytes" "flag" + "fmt" "os" "runtime" "testing" @@ -10,6 +11,8 @@ import ( "github.com/stretchr/testify/require" "github.com/tendermint/tendermint/libs/db" + mathrand "math/rand" + cmn "github.com/tendermint/tendermint/libs/common" ) @@ -45,7 +48,7 @@ func TestVersionedRandomTree(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree := NewVersionedTree(d, 100) + tree := NewMutableTree(d, 100) versions := 50 keysPerVersion := 30 @@ -70,7 +73,9 @@ func TestVersionedRandomTree(t *testing.T) { } require.Len(tree.versions, 1, "tree must have one version left") - require.Equal(tree.versions[int64(versions)].root, tree.root) + tr, err := tree.GetImmutable(int64(versions)) + require.NoError(err, "GetImmutable should not error for version %d", versions) + require.Equal(tr.root, tree.root) // After cleaning up all previous versions, we should have as many nodes // in the db as in the current tree version. @@ -84,8 +89,8 @@ func TestVersionedRandomTreeSmallKeys(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree := NewVersionedTree(d, 100) - singleVersionTree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(d, 100) + singleVersionTree := NewMutableTree(db.NewMemDB(), 0) versions := 20 keysPerVersion := 50 @@ -125,8 +130,8 @@ func TestVersionedRandomTreeSmallKeysRandomDeletes(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree := NewVersionedTree(d, 100) - singleVersionTree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(d, 100) + singleVersionTree := NewMutableTree(db.NewMemDB(), 0) versions := 30 keysPerVersion := 50 @@ -162,7 +167,7 @@ func TestVersionedRandomTreeSmallKeysRandomDeletes(t *testing.T) { } func TestVersionedTreeSpecial1(t *testing.T) { - tree := NewVersionedTree(db.NewMemDB(), 100) + tree := NewMutableTree(db.NewMemDB(), 100) tree.Set([]byte("C"), []byte("so43QQFN")) tree.SaveVersion() @@ -185,7 +190,7 @@ func TestVersionedTreeSpecial1(t *testing.T) { func TestVersionedRandomTreeSpecial2(t *testing.T) { require := require.New(t) - tree := NewVersionedTree(db.NewMemDB(), 100) + tree := NewMutableTree(db.NewMemDB(), 100) tree.Set([]byte("OFMe2Yvm"), []byte("ez2OtQtE")) tree.Set([]byte("WEN4iN7Y"), []byte("kQNyUalI")) @@ -204,7 +209,7 @@ func TestVersionedEmptyTree(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree := NewVersionedTree(d, 0) + tree := NewMutableTree(d, 0) hash, v, err := tree.SaveVersion() require.Nil(hash) @@ -242,14 +247,17 @@ func TestVersionedEmptyTree(t *testing.T) { // Now reload the tree. - tree = NewVersionedTree(d, 0) + tree = NewMutableTree(d, 0) tree.Load() require.False(tree.VersionExists(1)) require.True(tree.VersionExists(2)) require.False(tree.VersionExists(3)) - require.Empty(tree.versions[2].root) + t2, err := tree.GetImmutable(2) + require.NoError(err, "GetImmutable should not fail for version 2") + + require.Empty(t2.root) } func TestVersionedTree(t *testing.T) { @@ -257,7 +265,7 @@ func TestVersionedTree(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree := NewVersionedTree(d, 0) + tree := NewMutableTree(d, 0) // We start with zero keys in the databse. require.Equal(0, tree.ndb.size()) @@ -301,7 +309,7 @@ func TestVersionedTree(t *testing.T) { // Recreate a new tree and load it, to make sure it works in this // scenario. - tree = NewVersionedTree(d, 100) + tree = NewMutableTree(d, 100) _, err = tree.Load() require.NoError(err) @@ -347,7 +355,7 @@ func TestVersionedTree(t *testing.T) { require.EqualValues(hash3, hash4) require.NotNil(hash4) - tree = NewVersionedTree(d, 100) + tree = NewMutableTree(d, 100) _, err = tree.Load() require.NoError(err) @@ -442,7 +450,7 @@ func TestVersionedTreeVersionDeletingEfficiency(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree := NewVersionedTree(d, 0) + tree := NewMutableTree(d, 0) tree.Set([]byte("key0"), []byte("val0")) tree.Set([]byte("key1"), []byte("val0")) @@ -473,7 +481,7 @@ func TestVersionedTreeVersionDeletingEfficiency(t *testing.T) { require.Len(t, tree.ndb.leafNodes(), 3) - tree2 := NewVersionedTree(db.NewMemDB(), 0) + tree2 := NewMutableTree(db.NewMemDB(), 0) tree2.Set([]byte("key0"), []byte("val2")) tree2.Set([]byte("key2"), []byte("val2")) tree2.Set([]byte("key3"), []byte("val1")) @@ -484,7 +492,7 @@ func TestVersionedTreeVersionDeletingEfficiency(t *testing.T) { func TestVersionedTreeOrphanDeleting(t *testing.T) { mdb := db.NewMemDB() - tree := NewVersionedTree(mdb, 0) + tree := NewMutableTree(mdb, 0) tree.Set([]byte("key0"), []byte("val0")) tree.Set([]byte("key1"), []byte("val0")) @@ -522,7 +530,7 @@ func TestVersionedTreeOrphanDeleting(t *testing.T) { func TestVersionedTreeSpecialCase(t *testing.T) { require := require.New(t) - tree := NewVersionedTree(db.NewMemDB(), 100) + tree := NewMutableTree(db.NewMemDB(), 100) tree.Set([]byte("key1"), []byte("val0")) tree.Set([]byte("key2"), []byte("val0")) @@ -545,7 +553,7 @@ func TestVersionedTreeSpecialCase2(t *testing.T) { require := require.New(t) d := db.NewMemDB() - tree := NewVersionedTree(d, 100) + tree := NewMutableTree(d, 100) tree.Set([]byte("key1"), []byte("val0")) tree.Set([]byte("key2"), []byte("val0")) @@ -558,7 +566,7 @@ func TestVersionedTreeSpecialCase2(t *testing.T) { tree.Set([]byte("key2"), []byte("val2")) tree.SaveVersion() - tree = NewVersionedTree(d, 100) + tree = NewMutableTree(d, 100) _, err := tree.Load() require.NoError(err) @@ -570,7 +578,7 @@ func TestVersionedTreeSpecialCase2(t *testing.T) { func TestVersionedTreeSpecialCase3(t *testing.T) { require := require.New(t) - tree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(db.NewMemDB(), 0) tree.Set([]byte("m"), []byte("liWT0U6G")) tree.Set([]byte("G"), []byte("7PxRXwUA")) @@ -599,7 +607,7 @@ func TestVersionedTreeSpecialCase3(t *testing.T) { func TestVersionedTreeSaveAndLoad(t *testing.T) { require := require.New(t) d := db.NewMemDB() - tree := NewVersionedTree(d, 0) + tree := NewMutableTree(d, 0) // Loading with an empty root is a no-op. tree.Load() @@ -623,7 +631,7 @@ func TestVersionedTreeSaveAndLoad(t *testing.T) { require.Equal(6, tree.Version()) // Reload the tree, to test that roots and orphans are properly loaded. - ntree := NewVersionedTree(d, 0) + ntree := NewMutableTree(d, 0) ntree.Load() require.False(ntree.IsEmpty()) @@ -649,7 +657,7 @@ func TestVersionedTreeSaveAndLoad(t *testing.T) { func TestVersionedTreeErrors(t *testing.T) { require := require.New(t) - tree := NewVersionedTree(db.NewMemDB(), 100) + tree := NewMutableTree(db.NewMemDB(), 100) // Can't delete non-existent versions. require.Error(tree.DeleteVersion(1)) @@ -681,7 +689,7 @@ func TestVersionedCheckpoints(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree := NewVersionedTree(d, 100) + tree := NewMutableTree(d, 100) versions := 50 keysPerVersion := 10 versionsPerCheckpoint := 5 @@ -734,7 +742,7 @@ func TestVersionedCheckpoints(t *testing.T) { func TestVersionedCheckpointsSpecialCase(t *testing.T) { require := require.New(t) - tree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(db.NewMemDB(), 0) key := []byte("k") tree.Set(key, []byte("val1")) @@ -759,7 +767,7 @@ func TestVersionedCheckpointsSpecialCase(t *testing.T) { } func TestVersionedCheckpointsSpecialCase2(t *testing.T) { - tree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(db.NewMemDB(), 0) tree.Set([]byte("U"), []byte("XamDUtiJ")) tree.Set([]byte("A"), []byte("UkZBuYIU")) @@ -779,7 +787,7 @@ func TestVersionedCheckpointsSpecialCase2(t *testing.T) { } func TestVersionedCheckpointsSpecialCase3(t *testing.T) { - tree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(db.NewMemDB(), 0) tree.Set([]byte("n"), []byte("2wUCUs8q")) tree.Set([]byte("l"), []byte("WQ7mvMbc")) @@ -799,7 +807,7 @@ func TestVersionedCheckpointsSpecialCase3(t *testing.T) { } func TestVersionedCheckpointsSpecialCase4(t *testing.T) { - tree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(db.NewMemDB(), 0) tree.Set([]byte("U"), []byte("XamDUtiJ")) tree.Set([]byte("A"), []byte("UkZBuYIU")) @@ -831,7 +839,7 @@ func TestVersionedCheckpointsSpecialCase4(t *testing.T) { } func TestVersionedCheckpointsSpecialCase5(t *testing.T) { - tree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(db.NewMemDB(), 0) tree.Set([]byte("R"), []byte("ygZlIzeW")) tree.SaveVersion() @@ -848,7 +856,7 @@ func TestVersionedCheckpointsSpecialCase5(t *testing.T) { } func TestVersionedCheckpointsSpecialCase6(t *testing.T) { - tree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(db.NewMemDB(), 0) tree.Set([]byte("Y"), []byte("MW79JQeV")) tree.Set([]byte("7"), []byte("Kp0ToUJB")) @@ -880,7 +888,7 @@ func TestVersionedCheckpointsSpecialCase6(t *testing.T) { } func TestVersionedCheckpointsSpecialCase7(t *testing.T) { - tree := NewVersionedTree(db.NewMemDB(), 100) + tree := NewMutableTree(db.NewMemDB(), 100) tree.Set([]byte("n"), []byte("OtqD3nyn")) tree.Set([]byte("W"), []byte("kMdhJjF5")) @@ -914,7 +922,7 @@ func TestVersionedCheckpointsSpecialCase7(t *testing.T) { func TestVersionedTreeEfficiency(t *testing.T) { require := require.New(t) - tree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(db.NewMemDB(), 0) versions := 20 keysPerVersion := 100 keysAddedPerVersion := map[int]int{} @@ -949,7 +957,7 @@ func TestVersionedTreeEfficiency(t *testing.T) { func TestVersionedTreeProofs(t *testing.T) { require := require.New(t) - tree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(db.NewMemDB(), 0) tree.Set([]byte("k1"), []byte("v1")) tree.Set([]byte("k2"), []byte("v1")) @@ -1017,9 +1025,41 @@ func TestVersionedTreeProofs(t *testing.T) { require.Error(proof.Verify(root2)) } +func TestOrphans(t *testing.T) { + //If you create a sequence of saved versions + //Then randomly delete versions other than the first and last until only those two remain + //Any remaining orphan nodes should be constrained to just the first version + require := require.New(t) + tree := NewMutableTree(db.NewMemDB(), 100) + + NUMVERSIONS := 100 + NUMUPDATES := 100 + + for i := 0; i < NUMVERSIONS; i++ { + for j := 1; j < NUMUPDATES; j++ { + tree.Set(randBytes(2), randBytes(2)) + } + _, _, err := tree.SaveVersion() + require.NoError(err, "SaveVersion should not error") + } + + idx := mathrand.Perm(NUMVERSIONS - 2) + for i := range idx { + err := tree.DeleteVersion(int64(i + 2)) + require.NoError(err, "DeleteVersion should not error") + } + + tree.ndb.traverseOrphans(func(k, v []byte) { + var fromVersion, toVersion int64 + fmt.Sscanf(string(k), orphanKeyFmt, &toVersion, &fromVersion) + require.Equal(fromVersion, int64(1), "fromVersion should be 1") + require.Equal(toVersion, int64(1), "toVersion should be 1") + }) +} + func TestVersionedTreeHash(t *testing.T) { require := require.New(t) - tree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(db.NewMemDB(), 0) require.Nil(tree.Hash()) tree.Set([]byte("I"), []byte("D")) @@ -1041,7 +1081,7 @@ func TestVersionedTreeHash(t *testing.T) { func TestNilValueSemantics(t *testing.T) { require := require.New(t) - tree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(db.NewMemDB(), 0) require.Panics(func() { tree.Set([]byte("k"), nil) @@ -1051,7 +1091,7 @@ func TestNilValueSemantics(t *testing.T) { func TestCopyValueSemantics(t *testing.T) { require := require.New(t) - tree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(db.NewMemDB(), 0) val := []byte("v1") @@ -1068,7 +1108,7 @@ func TestCopyValueSemantics(t *testing.T) { func TestRollback(t *testing.T) { require := require.New(t) - tree := NewVersionedTree(db.NewMemDB(), 0) + tree := NewMutableTree(db.NewMemDB(), 0) tree.Set([]byte("k"), []byte("v")) tree.SaveVersion() @@ -1094,6 +1134,38 @@ func TestRollback(t *testing.T) { require.Equal([]byte("v"), val) } +func TestOverwrite(t *testing.T) { + require := require.New(t) + + mdb := db.NewMemDB() + tree := NewMutableTree(mdb, 0) + + // Set one kv pair and save version 1 + tree.Set([]byte("key1"), []byte("value1")) + _, _, err := tree.SaveVersion() + require.NoError(err, "SaveVersion should not fail") + + // Set another kv pair and save version 2 + tree.Set([]byte("key2"), []byte("value2")) + _, _, err = tree.SaveVersion() + require.NoError(err, "SaveVersion should not fail") + + // Reload tree at version 1 + tree = NewMutableTree(mdb, 0) + _, err = tree.LoadVersion(int64(1)) + require.NoError(err, "LoadVersion should not fail") + + // Attempt to put a different kv pair into the tree and save + tree.Set([]byte("key2"), []byte("different value 2")) + _, _, err = tree.SaveVersion() + require.Error(err, "SaveVersion should fail because of changed value") + + // Replay the original transition from version 1 to version 2 and attempt to save + tree.Set([]byte("key2"), []byte("value2")) + _, _, err = tree.SaveVersion() + require.NoError(err, "SaveVersion should not fail, overwrite was idempotent") +} + //////////////////////////// BENCHMARKS /////////////////////////////////////// func BenchmarkTreeLoadAndDelete(b *testing.B) { @@ -1107,7 +1179,7 @@ func BenchmarkTreeLoadAndDelete(b *testing.B) { defer d.Close() defer os.RemoveAll("./bench.db") - tree := NewVersionedTree(d, 0) + tree := NewMutableTree(d, 0) for v := 1; v < numVersions; v++ { for i := 0; i < numKeysPerVersion; i++ { tree.Set([]byte(rand.Str(16)), rand.Bytes(32)) @@ -1118,7 +1190,7 @@ func BenchmarkTreeLoadAndDelete(b *testing.B) { b.Run("LoadAndDelete", func(b *testing.B) { for n := 0; n < b.N; n++ { b.StopTimer() - tree = NewVersionedTree(d, 0) + tree = NewMutableTree(d, 0) runtime.GC() b.StartTimer() diff --git a/util.go b/util.go index 96f754189..b2e7facfe 100644 --- a/util.go +++ b/util.go @@ -7,7 +7,7 @@ import ( ) // PrintTree prints the whole tree in an indented form. -func PrintTree(tree *Tree) { +func PrintTree(tree *ImmutableTree) { ndb, root := tree.ndb, tree.root printNode(ndb, root, 0) } diff --git a/versioned_tree.go b/versioned_tree.go deleted file mode 100644 index 7d8108ea8..000000000 --- a/versioned_tree.go +++ /dev/null @@ -1,197 +0,0 @@ -package iavl - -import ( - "bytes" - "fmt" - - cmn "github.com/tendermint/tendermint/libs/common" - dbm "github.com/tendermint/tendermint/libs/db" -) - -// ErrVersionDoesNotExist is returned if a requested version does not exist. -var ErrVersionDoesNotExist = fmt.Errorf("version does not exist") - -// VersionedTree is a persistent tree which keeps track of versions. -type VersionedTree struct { - *orphaningTree // The current, working tree. - versions map[int64]*Tree // The previous, saved versions of the tree. - ndb *nodeDB -} - -// NewVersionedTree returns a new tree with the specified cache size and datastore. -func NewVersionedTree(db dbm.DB, cacheSize int) *VersionedTree { - ndb := newNodeDB(db, cacheSize) - head := &Tree{ndb: ndb} - - return &VersionedTree{ - orphaningTree: newOrphaningTree(head), - versions: map[int64]*Tree{}, - ndb: ndb, - } -} - -// IsEmpty returns whether or not the tree has any keys. Only trees that are -// not empty can be saved. -func (tree *VersionedTree) IsEmpty() bool { - return tree.orphaningTree.Size() == 0 -} - -// VersionExists returns whether or not a version exists. -func (tree *VersionedTree) VersionExists(version int64) bool { - _, ok := tree.versions[version] - return ok -} - -// Tree returns the current working tree. -func (tree *VersionedTree) Tree() *Tree { - return tree.orphaningTree.Tree -} - -// Hash returns the hash of the latest saved version of the tree, as returned -// by SaveVersion. If no versions have been saved, Hash returns nil. -func (tree *VersionedTree) Hash() []byte { - if tree.version > 0 { - return tree.versions[tree.version].Hash() - } - return nil -} - -// String returns a string representation of the tree. -func (tree *VersionedTree) String() string { - return tree.ndb.String() -} - -// Set sets a key in the working tree. Nil values are not supported. -func (tree *VersionedTree) Set(key, val []byte) bool { - return tree.orphaningTree.Set(key, val) -} - -// Remove removes a key from the working tree. -func (tree *VersionedTree) Remove(key []byte) ([]byte, bool) { - return tree.orphaningTree.Remove(key) -} - -// Load the latest versioned tree from disk. -// -// Returns the version number of the latest version found -func (tree *VersionedTree) Load() (int64, error) { - return tree.LoadVersion(0) -} - -// Load a versioned tree from disk. -// -// If version is 0, the latest version is loaded. -// -// Returns the version number of the latest version found -func (tree *VersionedTree) LoadVersion(targetVersion int64) (int64, error) { - roots, err := tree.ndb.getRoots() - if err != nil { - return 0, err - } - if len(roots) == 0 { - return 0, nil - } - - // Load all roots from the database. - latestVersion := int64(0) - for version, root := range roots { - - // Construct a tree manually. - t := &Tree{} - t.ndb = tree.ndb - t.version = version - if len(root) != 0 { - t.root = tree.ndb.GetNode(root) - } - tree.versions[version] = t - - if version > latestVersion && - (targetVersion == 0 || version <= targetVersion) { - - latestVersion = version - } - } - - // Validate latestVersion - if !(targetVersion == 0 || latestVersion == targetVersion) { - return latestVersion, fmt.Errorf("Wanted to load target %v but only found up to %v", - targetVersion, latestVersion) - } - - // Set the working tree to a copy of the latest. - tree.orphaningTree = newOrphaningTree( - tree.versions[latestVersion].clone(), - ) - - return latestVersion, nil -} - -// Rollback resets the working tree to the latest saved version, discarding -// any unsaved modifications. -func (tree *VersionedTree) Rollback() { - if tree.version > 0 { - tree.orphaningTree = newOrphaningTree( - tree.versions[tree.version].clone(), - ) - } else { - tree.orphaningTree = newOrphaningTree(&Tree{ndb: tree.ndb, version: 0}) - } -} - -// GetVersioned gets the value at the specified key and version. -func (tree *VersionedTree) GetVersioned(key []byte, version int64) ( - index int, value []byte, -) { - if t, ok := tree.versions[version]; ok { - return t.Get(key) - } - return -1, nil -} - -// SaveVersion saves a new tree version to disk, based on the current state of -// the tree. Returns the hash and new version number. -func (tree *VersionedTree) SaveVersion() ([]byte, int64, error) { - version := tree.version + 1 - - if _, ok := tree.versions[version]; ok { - // Same hash means idempotent. Return success. - var existingHash = tree.versions[version].Hash() - var newHash = tree.orphaningTree.Hash() - if bytes.Equal(existingHash, newHash) { - tree.orphaningTree = newOrphaningTree(tree.versions[version].clone()) - return existingHash, version, nil - } - return nil, version, fmt.Errorf("version %d was already saved to different hash %X (existing hash %X)", - version, newHash, existingHash) - } - - // Persist version and stash to .versions. - tree.orphaningTree.SaveAs(version) - tree.versions[version] = tree.orphaningTree.Tree - - // Set new working tree. - tree.orphaningTree = newOrphaningTree(tree.orphaningTree.clone()) - - return tree.Hash(), version, nil -} - -// DeleteVersion deletes a tree version from disk. The version can then no -// longer be accessed. -func (tree *VersionedTree) DeleteVersion(version int64) error { - if version == 0 { - return cmn.NewError("version must be greater than 0") - } - if version == tree.version { - return cmn.NewError("cannot delete latest saved version (%d)", version) - } - if _, ok := tree.versions[version]; !ok { - return cmn.ErrorWrap(ErrVersionDoesNotExist, "") - } - - tree.ndb.DeleteVersion(version) - tree.ndb.Commit() - - delete(tree.versions, version) - - return nil -} From 9e62436856efa94c1223043be36ebda01ae0b6fc Mon Sep 17 00:00:00 2001 From: Ismail Khoffi Date: Tue, 14 Aug 2018 12:01:35 +0100 Subject: [PATCH 09/10] delete empty file (relict from merging master into develop) --- versioned_tree.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 versioned_tree.go diff --git a/versioned_tree.go b/versioned_tree.go deleted file mode 100644 index e69de29bb..000000000 From 26179c24dd191121ceb43a376d219e30eeaa4511 Mon Sep 17 00:00:00 2001 From: Ismail Khoffi Date: Tue, 14 Aug 2018 12:34:32 +0100 Subject: [PATCH 10/10] Release 0.10.0: Update Changelog and bump version (#99) --- CHANGELOG.md | 18 ++++++++++++++++-- version.go | 2 +- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b800a6b9..46e3f5ca4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,24 @@ # Changelog -## PENDING +## 0.10.0 + +BREAKING CHANGES + +- refactored API for clean separation of [mutable][1] and [immutable][2] tree (#92, #88); +with possibility to: + - load read-only snapshots at previous versions on demand + - load mutable trees at the most recently saved tree + +[1]: https://github.com/tendermint/iavl/blob/9e62436856efa94c1223043be36ebda01ae0b6fc/mutable_tree.go#L14-L21 +[2]: https://github.com/tendermint/iavl/blob/9e62436856efa94c1223043be36ebda01ae0b6fc/immutable_tree.go#L10-L17 + +BUG FIXES + +- remove memory leaks (#92) IMPROVEMENTS -- Change tendermint dep to ^v0.22.0 +- Change tendermint dep to ^v0.22.0 (#91) ## 0.9.2 (July 3, 2018) diff --git a/version.go b/version.go index 9efd11145..f88f00abc 100644 --- a/version.go +++ b/version.go @@ -1,4 +1,4 @@ package iavl // Version of iavl. -const Version = "0.9.2" +const Version = "0.10.0"