Skip to content

Commit

Permalink
use configured tempdir in branchEncoder (#10535)
Browse files Browse the repository at this point in the history
  • Loading branch information
@AskAlexSharov
AskAlexSharov authored May 29, 2024
1 parent 4604ad0 commit afcd1b0
Show file tree
Hide file tree
Showing 7 changed files with 45 additions and 57 deletions.
11 changes: 2 additions & 9 deletions erigon-lib/commitment/bin_patricia_hashed.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (
"fmt"
"io"
"math/bits"
"os"
"path/filepath"
"sort"

Expand Down Expand Up @@ -119,7 +118,7 @@ type BinPatriciaHashed struct {
storageFn func(plainKey []byte, cell *BinaryCell) error
}

func NewBinPatriciaHashed(accountKeyLen int, ctx PatriciaContext) *BinPatriciaHashed {
func NewBinPatriciaHashed(accountKeyLen int, ctx PatriciaContext, tmpdir string) *BinPatriciaHashed {
bph := &BinPatriciaHashed{
keccak: sha3.NewLegacyKeccak256().(keccakState),
keccak2: sha3.NewLegacyKeccak256().(keccakState),
Expand All @@ -129,13 +128,7 @@ func NewBinPatriciaHashed(accountKeyLen int, ctx PatriciaContext) *BinPatriciaHa
auxBuffer: bytes.NewBuffer(make([]byte, 8192)),
ctx: ctx,
}
tdir := os.TempDir()
if ctx != nil {
tdir = ctx.TempDir()
}

tdir = filepath.Join(tdir, "branch-encoder")
bph.branchEncoder = NewBranchEncoder(1024, tdir)
bph.branchEncoder = NewBranchEncoder(1024, filepath.Join(tmpdir, "branch-encoder"))

return bph

Expand Down
12 changes: 6 additions & 6 deletions erigon-lib/commitment/bin_patricia_hashed_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ func Test_BinPatriciaTrie_UniqueRepresentation(t *testing.T) {
ms := NewMockState(t)
ms2 := NewMockState(t)

trie := NewBinPatriciaHashed(length.Addr, ms)
trieBatch := NewBinPatriciaHashed(length.Addr, ms2)
trie := NewBinPatriciaHashed(length.Addr, ms, ms.TempDir())
trieBatch := NewBinPatriciaHashed(length.Addr, ms2, ms2.TempDir())

plainKeys, updates := NewUpdateBuilder().
Balance("e25652aaa6b9417973d325f9a1246b48ff9420bf", 12).
Expand Down Expand Up @@ -109,8 +109,8 @@ func Test_BinPatriciaHashed_UniqueRepresentation(t *testing.T) {
Storage("f5", "04", "9898").
Build()

trieOne := NewBinPatriciaHashed(1, ms)
trieTwo := NewBinPatriciaHashed(1, ms2)
trieOne := NewBinPatriciaHashed(1, ms, ms.TempDir())
trieTwo := NewBinPatriciaHashed(1, ms2, ms2.TempDir())

trieOne.SetTrace(true)
trieTwo.SetTrace(true)
Expand Down Expand Up @@ -155,7 +155,7 @@ func Test_BinPatriciaHashed_UniqueRepresentation(t *testing.T) {
func Test_BinPatriciaHashed_EmptyState(t *testing.T) {
ctx := context.Background()
ms := NewMockState(t)
hph := NewBinPatriciaHashed(1, ms)
hph := NewBinPatriciaHashed(1, ms, ms.TempDir())
hph.SetTrace(false)
plainKeys, updates := NewUpdateBuilder().
Balance("00", 4).
Expand Down Expand Up @@ -222,7 +222,7 @@ func Test_BinPatriciaHashed_EmptyState(t *testing.T) {
func Test_BinPatriciaHashed_EmptyUpdateState(t *testing.T) {
ctx := context.Background()
ms := NewMockState(t)
hph := NewBinPatriciaHashed(1, ms)
hph := NewBinPatriciaHashed(1, ms, ms.TempDir())
hph.SetTrace(false)
plainKeys, updates := NewUpdateBuilder().
Balance("00", 4).
Expand Down
12 changes: 6 additions & 6 deletions erigon-lib/commitment/commitment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,16 @@ import (
"context"
"encoding/binary"
"fmt"
"math/bits"
"strings"

"github.com/google/btree"
"github.com/ledgerwatch/erigon-lib/common"
"github.com/ledgerwatch/erigon-lib/common/cryptozerocopy"
"github.com/ledgerwatch/erigon-lib/metrics"
"github.com/ledgerwatch/erigon-lib/types"
"github.com/ledgerwatch/log/v3"
"golang.org/x/crypto/sha3"
"math/bits"
"strings"

"github.com/ledgerwatch/erigon-lib/common/length"
"github.com/ledgerwatch/erigon-lib/etl"
Expand Down Expand Up @@ -59,8 +60,6 @@ type PatriciaContext interface {
GetAccount(plainKey []byte, cell *Cell) error
// fetch storage with given plain key
GetStorage(plainKey []byte, cell *Cell) error
// Returns temp directory to use for update collecting
TempDir() string
// store branch data
PutBranch(prefix []byte, data []byte, prevData []byte, prevStep uint64) error
}
Expand All @@ -77,14 +76,15 @@ const (
func InitializeTrieAndUpdateTree(tv TrieVariant, mode Mode, tmpdir string) (Trie, *UpdateTree) {
switch tv {
case VariantBinPatriciaTrie:
trie := NewBinPatriciaHashed(length.Addr, nil)
trie := NewBinPatriciaHashed(length.Addr, nil, tmpdir)
fn := func(key []byte) []byte { return hexToBin(key) }
tree := NewUpdateTree(mode, tmpdir, fn)
return trie, tree
case VariantHexPatriciaTrie:
fallthrough
default:
trie := NewHexPatriciaHashed(length.Addr, nil)

trie := NewHexPatriciaHashed(length.Addr, nil, tmpdir)
tree := NewUpdateTree(mode, tmpdir, trie.hashAndNibblizeKey)
return trie, tree
}
Expand Down
13 changes: 4 additions & 9 deletions erigon-lib/commitment/hex_patricia_hashed.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,17 +22,17 @@ import (
"encoding/binary"
"encoding/hex"
"fmt"
"github.com/ledgerwatch/erigon-lib/etl"
"hash"
"io"
"math/bits"
"os"
"path/filepath"
"runtime"
"sort"
"strings"
"time"

"github.com/ledgerwatch/erigon-lib/etl"

"github.com/ledgerwatch/erigon-lib/common/dbg"

"github.com/ledgerwatch/log/v3"
Expand Down Expand Up @@ -85,20 +85,15 @@ type HexPatriciaHashed struct {
branchEncoder *BranchEncoder
}

func NewHexPatriciaHashed(accountKeyLen int, ctx PatriciaContext) *HexPatriciaHashed {
func NewHexPatriciaHashed(accountKeyLen int, ctx PatriciaContext, tmpdir string) *HexPatriciaHashed {
hph := &HexPatriciaHashed{
ctx: ctx,
keccak: sha3.NewLegacyKeccak256().(keccakState),
keccak2: sha3.NewLegacyKeccak256().(keccakState),
accountKeyLen: accountKeyLen,
auxBuffer: bytes.NewBuffer(make([]byte, 8192)),
}
tdir := os.TempDir()
if ctx != nil {
tdir = ctx.TempDir()
}
tdir = filepath.Join(tdir, "branch-encoder")
hph.branchEncoder = NewBranchEncoder(1024, tdir)
hph.branchEncoder = NewBranchEncoder(1024, filepath.Join(tmpdir, "branch-encoder"))
return hph
}

Expand Down
2 changes: 1 addition & 1 deletion erigon-lib/commitment/hex_patricia_hashed_bench_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (
func Benchmark_HexPatriciaHashed_ReviewKeys(b *testing.B) {
ms := NewMockState(&testing.T{})
ctx := context.Background()
hph := NewHexPatriciaHashed(length.Addr, ms)
hph := NewHexPatriciaHashed(length.Addr, ms, ms.TempDir())
hph.SetTrace(false)

builder := NewUpdateBuilder()
Expand Down
10 changes: 5 additions & 5 deletions erigon-lib/commitment/hex_patricia_hashed_fuzz_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,8 @@ func Fuzz_ProcessUpdate(f *testing.F) {

ms := NewMockState(t)
ms2 := NewMockState(t)
hph := NewHexPatriciaHashed(20, ms)
hphAnother := NewHexPatriciaHashed(20, ms2)
hph := NewHexPatriciaHashed(20, ms, ms.TempDir())
hphAnother := NewHexPatriciaHashed(20, ms2, ms2.TempDir())

hph.SetTrace(false)
hphAnother.SetTrace(false)
Expand Down Expand Up @@ -142,8 +142,8 @@ func Fuzz_ProcessUpdates_ArbitraryUpdateCount(f *testing.F) {

ms := NewMockState(t)
ms2 := NewMockState(t)
hph := NewHexPatriciaHashed(20, ms)
hphAnother := NewHexPatriciaHashed(20, ms2)
hph := NewHexPatriciaHashed(20, ms, ms.TempDir())
hphAnother := NewHexPatriciaHashed(20, ms2, ms2.TempDir())

plainKeys, updates := builder.Build()

Expand Down Expand Up @@ -199,7 +199,7 @@ func Fuzz_HexPatriciaHashed_ReviewKeys(f *testing.F) {
}

ms := NewMockState(t)
hph := NewHexPatriciaHashed(length.Addr, ms)
hph := NewHexPatriciaHashed(length.Addr, ms, ms.TempDir())

hph.SetTrace(false)

Expand Down
42 changes: 21 additions & 21 deletions erigon-lib/commitment/hex_patricia_hashed_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ import (
func Test_HexPatriciaHashed_ResetThenSingularUpdates(t *testing.T) {
ctx := context.Background()
ms := NewMockState(t)
hph := NewHexPatriciaHashed(1, ms)
hph := NewHexPatriciaHashed(1, ms, ms.TempDir())
hph.SetTrace(false)
plainKeys, updates := NewUpdateBuilder().
Balance("00", 4).
Expand Down Expand Up @@ -107,7 +107,7 @@ func Test_HexPatriciaHashed_ResetThenSingularUpdates(t *testing.T) {
func Test_HexPatriciaHashed_EmptyUpdate(t *testing.T) {
ms := NewMockState(t)
ctx := context.Background()
hph := NewHexPatriciaHashed(1, ms)
hph := NewHexPatriciaHashed(1, ms, ms.TempDir())
hph.SetTrace(false)
plainKeys, updates := NewUpdateBuilder().
Balance("00", 4).
Expand Down Expand Up @@ -161,8 +161,8 @@ func Test_HexPatriciaHashed_UniqueRepresentation2(t *testing.T) {
//Balance("0000000000000000000000000000000000000000", 4000000000000138901).
Build()

trieOne := NewHexPatriciaHashed(20, ms)
trieTwo := NewHexPatriciaHashed(20, ms2)
trieOne := NewHexPatriciaHashed(20, ms, ms.TempDir())
trieTwo := NewHexPatriciaHashed(20, ms2, ms2.TempDir())

//trieOne.SetTrace(true)
//trieTwo.SetTrace(true)
Expand Down Expand Up @@ -298,8 +298,8 @@ func Test_HexPatriciaHashed_BrokenUniqueRepr(t *testing.T) {
Build()

keyLen := 20
trieSequential := NewHexPatriciaHashed(keyLen, stateSeq)
trieBatch := NewHexPatriciaHashed(keyLen, stateBatch)
trieSequential := NewHexPatriciaHashed(keyLen, stateSeq, stateSeq.TempDir())
trieBatch := NewHexPatriciaHashed(keyLen, stateBatch, stateBatch.TempDir())

if sortHashedKeys {
plainKeys, updates = sortUpdatesByHashIncrease(t, trieSequential, plainKeys, updates)
Expand Down Expand Up @@ -387,8 +387,8 @@ func Test_HexPatriciaHashed_UniqueRepresentation(t *testing.T) {
Storage("68ee6c0e9cdc73b2b2d52dbd79f19d24fe25e2f9", "d1664244ae1a8a05f8f1d41e45548fbb7aa54609b985d6439ee5fd9bb0da619f", "9898").
Build()

trieSequential := NewHexPatriciaHashed(length.Addr, stateSeq)
trieBatch := NewHexPatriciaHashed(length.Addr, stateBatch)
trieSequential := NewHexPatriciaHashed(length.Addr, stateSeq, stateSeq.TempDir())
trieBatch := NewHexPatriciaHashed(length.Addr, stateBatch, stateBatch.TempDir())

plainKeys, updates = sortUpdatesByHashIncrease(t, trieSequential, plainKeys, updates)

Expand Down Expand Up @@ -477,7 +477,7 @@ func Test_HexPatriciaHashed_Sepolia(t *testing.T) {
},
}

hph := NewHexPatriciaHashed(length.Addr, ms)
hph := NewHexPatriciaHashed(length.Addr, ms, ms.TempDir())
//hph.SetTrace(true)

for _, testData := range tests {
Expand Down Expand Up @@ -611,8 +611,8 @@ func Test_HexPatriciaHashed_StateEncodeDecodeSetup(t *testing.T) {
Storage("f5", "04", "9898").
Build()

before := NewHexPatriciaHashed(1, ms)
after := NewHexPatriciaHashed(1, ms)
before := NewHexPatriciaHashed(1, ms, ms.TempDir())
after := NewHexPatriciaHashed(1, ms, ms.TempDir())

err := ms.applyPlainUpdates(plainKeys, updates)
require.NoError(t, err)
Expand Down Expand Up @@ -659,7 +659,7 @@ func Test_HexPatriciaHashed_StateRestoreAndContinue(t *testing.T) {
Balance("ff", 900234).
Build()

trieOne := NewHexPatriciaHashed(1, ms)
trieOne := NewHexPatriciaHashed(1, ms, ms.TempDir())
err := ms.applyPlainUpdates(plainKeys, updates)
require.NoError(t, err)
err = ms2.applyPlainUpdates(plainKeys, updates)
Expand All @@ -681,7 +681,7 @@ func Test_HexPatriciaHashed_StateRestoreAndContinue(t *testing.T) {
require.NotEmpty(t, buf)

t.Logf("restore state to another trie\n")
trieTwo := NewHexPatriciaHashed(1, ms2)
trieTwo := NewHexPatriciaHashed(1, ms2, ms2.TempDir())
err = trieTwo.SetState(buf)
require.NoError(t, err)

Expand Down Expand Up @@ -744,8 +744,8 @@ func Test_HexPatriciaHashed_RestoreAndContinue(t *testing.T) {
Storage("f5", "04", "9898").
Build()

trieOne := NewHexPatriciaHashed(1, ms)
trieTwo := NewHexPatriciaHashed(1, ms)
trieOne := NewHexPatriciaHashed(1, ms, ms.TempDir())
trieTwo := NewHexPatriciaHashed(1, ms, ms.TempDir())

err := ms.applyPlainUpdates(plainKeys, updates)
require.NoError(t, err)
Expand Down Expand Up @@ -796,8 +796,8 @@ func Test_HexPatriciaHashed_ProcessUpdates_UniqueRepresentation_AfterStateRestor
Storage("68ee6c0e9cdc73b2b2d52dbd79f19d24fe25e2f9", "d1664244ae1a8a05f8f1d41e45548fbb7aa54609b985d6439ee5fd9bb0da619f", "9898").
Build()

sequential := NewHexPatriciaHashed(20, seqState)
batch := NewHexPatriciaHashed(20, batchState)
sequential := NewHexPatriciaHashed(20, seqState, seqState.TempDir())
batch := NewHexPatriciaHashed(20, batchState, batchState.TempDir())

plainKeys, updates = sortUpdatesByHashIncrease(t, sequential, plainKeys, updates)

Expand Down Expand Up @@ -826,7 +826,7 @@ func Test_HexPatriciaHashed_ProcessUpdates_UniqueRepresentation_AfterStateRestor
require.NoError(t, err)

sequential.Reset()
sequential = NewHexPatriciaHashed(20, seqState)
sequential = NewHexPatriciaHashed(20, seqState, seqState.TempDir())

err = sequential.SetState(prevState)
require.NoError(t, err)
Expand Down Expand Up @@ -895,8 +895,8 @@ func Test_HexPatriciaHashed_ProcessUpdates_UniqueRepresentationInTheMiddle(t *te
Balance("ba7a3b7b095d3370c022ca655c790f0c0ead66f5", 5*1e17).
Build()

sequential := NewHexPatriciaHashed(20, seqState)
batch := NewHexPatriciaHashed(20, batchState)
sequential := NewHexPatriciaHashed(20, seqState, seqState.TempDir())
batch := NewHexPatriciaHashed(20, batchState, batchState.TempDir())

plainKeys, updates = sortUpdatesByHashIncrease(t, sequential, plainKeys, updates)

Expand Down Expand Up @@ -927,7 +927,7 @@ func Test_HexPatriciaHashed_ProcessUpdates_UniqueRepresentationInTheMiddle(t *te
require.NoError(t, err)

sequential.Reset()
sequential = NewHexPatriciaHashed(20, seqState)
sequential = NewHexPatriciaHashed(20, seqState, seqState.TempDir())

err = sequential.SetState(prevState)
require.NoError(t, err)
Expand Down

0 comments on commit afcd1b0

Please sign in to comment.