From afcd1b006ca1dc08889500c5415a00f1a86f9b7e Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Wed, 29 May 2024 17:13:48 +0700 Subject: [PATCH] use configured tempdir in branchEncoder (#10535) --- erigon-lib/commitment/bin_patricia_hashed.go | 11 +---- .../commitment/bin_patricia_hashed_test.go | 12 +++--- erigon-lib/commitment/commitment.go | 12 +++--- erigon-lib/commitment/hex_patricia_hashed.go | 13 ++---- .../hex_patricia_hashed_bench_test.go | 2 +- .../hex_patricia_hashed_fuzz_test.go | 10 ++--- .../commitment/hex_patricia_hashed_test.go | 42 +++++++++---------- 7 files changed, 45 insertions(+), 57 deletions(-) diff --git a/erigon-lib/commitment/bin_patricia_hashed.go b/erigon-lib/commitment/bin_patricia_hashed.go index 33deb3a9826..321a97ee61d 100644 --- a/erigon-lib/commitment/bin_patricia_hashed.go +++ b/erigon-lib/commitment/bin_patricia_hashed.go @@ -24,7 +24,6 @@ import ( "fmt" "io" "math/bits" - "os" "path/filepath" "sort" @@ -119,7 +118,7 @@ type BinPatriciaHashed struct { storageFn func(plainKey []byte, cell *BinaryCell) error } -func NewBinPatriciaHashed(accountKeyLen int, ctx PatriciaContext) *BinPatriciaHashed { +func NewBinPatriciaHashed(accountKeyLen int, ctx PatriciaContext, tmpdir string) *BinPatriciaHashed { bph := &BinPatriciaHashed{ keccak: sha3.NewLegacyKeccak256().(keccakState), keccak2: sha3.NewLegacyKeccak256().(keccakState), @@ -129,13 +128,7 @@ func NewBinPatriciaHashed(accountKeyLen int, ctx PatriciaContext) *BinPatriciaHa auxBuffer: bytes.NewBuffer(make([]byte, 8192)), ctx: ctx, } - tdir := os.TempDir() - if ctx != nil { - tdir = ctx.TempDir() - } - - tdir = filepath.Join(tdir, "branch-encoder") - bph.branchEncoder = NewBranchEncoder(1024, tdir) + bph.branchEncoder = NewBranchEncoder(1024, filepath.Join(tmpdir, "branch-encoder")) return bph diff --git a/erigon-lib/commitment/bin_patricia_hashed_test.go b/erigon-lib/commitment/bin_patricia_hashed_test.go index 12e4404a62f..e5369fcb225 100644 --- a/erigon-lib/commitment/bin_patricia_hashed_test.go +++ b/erigon-lib/commitment/bin_patricia_hashed_test.go @@ -19,8 +19,8 @@ func Test_BinPatriciaTrie_UniqueRepresentation(t *testing.T) { ms := NewMockState(t) ms2 := NewMockState(t) - trie := NewBinPatriciaHashed(length.Addr, ms) - trieBatch := NewBinPatriciaHashed(length.Addr, ms2) + trie := NewBinPatriciaHashed(length.Addr, ms, ms.TempDir()) + trieBatch := NewBinPatriciaHashed(length.Addr, ms2, ms2.TempDir()) plainKeys, updates := NewUpdateBuilder(). Balance("e25652aaa6b9417973d325f9a1246b48ff9420bf", 12). @@ -109,8 +109,8 @@ func Test_BinPatriciaHashed_UniqueRepresentation(t *testing.T) { Storage("f5", "04", "9898"). Build() - trieOne := NewBinPatriciaHashed(1, ms) - trieTwo := NewBinPatriciaHashed(1, ms2) + trieOne := NewBinPatriciaHashed(1, ms, ms.TempDir()) + trieTwo := NewBinPatriciaHashed(1, ms2, ms2.TempDir()) trieOne.SetTrace(true) trieTwo.SetTrace(true) @@ -155,7 +155,7 @@ func Test_BinPatriciaHashed_UniqueRepresentation(t *testing.T) { func Test_BinPatriciaHashed_EmptyState(t *testing.T) { ctx := context.Background() ms := NewMockState(t) - hph := NewBinPatriciaHashed(1, ms) + hph := NewBinPatriciaHashed(1, ms, ms.TempDir()) hph.SetTrace(false) plainKeys, updates := NewUpdateBuilder(). Balance("00", 4). @@ -222,7 +222,7 @@ func Test_BinPatriciaHashed_EmptyState(t *testing.T) { func Test_BinPatriciaHashed_EmptyUpdateState(t *testing.T) { ctx := context.Background() ms := NewMockState(t) - hph := NewBinPatriciaHashed(1, ms) + hph := NewBinPatriciaHashed(1, ms, ms.TempDir()) hph.SetTrace(false) plainKeys, updates := NewUpdateBuilder(). Balance("00", 4). diff --git a/erigon-lib/commitment/commitment.go b/erigon-lib/commitment/commitment.go index 55e0dd81cc6..4aa628f06af 100644 --- a/erigon-lib/commitment/commitment.go +++ b/erigon-lib/commitment/commitment.go @@ -5,6 +5,9 @@ import ( "context" "encoding/binary" "fmt" + "math/bits" + "strings" + "github.com/google/btree" "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/cryptozerocopy" @@ -12,8 +15,6 @@ import ( "github.com/ledgerwatch/erigon-lib/types" "github.com/ledgerwatch/log/v3" "golang.org/x/crypto/sha3" - "math/bits" - "strings" "github.com/ledgerwatch/erigon-lib/common/length" "github.com/ledgerwatch/erigon-lib/etl" @@ -59,8 +60,6 @@ type PatriciaContext interface { GetAccount(plainKey []byte, cell *Cell) error // fetch storage with given plain key GetStorage(plainKey []byte, cell *Cell) error - // Returns temp directory to use for update collecting - TempDir() string // store branch data PutBranch(prefix []byte, data []byte, prevData []byte, prevStep uint64) error } @@ -77,14 +76,15 @@ const ( func InitializeTrieAndUpdateTree(tv TrieVariant, mode Mode, tmpdir string) (Trie, *UpdateTree) { switch tv { case VariantBinPatriciaTrie: - trie := NewBinPatriciaHashed(length.Addr, nil) + trie := NewBinPatriciaHashed(length.Addr, nil, tmpdir) fn := func(key []byte) []byte { return hexToBin(key) } tree := NewUpdateTree(mode, tmpdir, fn) return trie, tree case VariantHexPatriciaTrie: fallthrough default: - trie := NewHexPatriciaHashed(length.Addr, nil) + + trie := NewHexPatriciaHashed(length.Addr, nil, tmpdir) tree := NewUpdateTree(mode, tmpdir, trie.hashAndNibblizeKey) return trie, tree } diff --git a/erigon-lib/commitment/hex_patricia_hashed.go b/erigon-lib/commitment/hex_patricia_hashed.go index 0f2414a7831..3cf8fae9505 100644 --- a/erigon-lib/commitment/hex_patricia_hashed.go +++ b/erigon-lib/commitment/hex_patricia_hashed.go @@ -22,17 +22,17 @@ import ( "encoding/binary" "encoding/hex" "fmt" - "github.com/ledgerwatch/erigon-lib/etl" "hash" "io" "math/bits" - "os" "path/filepath" "runtime" "sort" "strings" "time" + "github.com/ledgerwatch/erigon-lib/etl" + "github.com/ledgerwatch/erigon-lib/common/dbg" "github.com/ledgerwatch/log/v3" @@ -85,7 +85,7 @@ type HexPatriciaHashed struct { branchEncoder *BranchEncoder } -func NewHexPatriciaHashed(accountKeyLen int, ctx PatriciaContext) *HexPatriciaHashed { +func NewHexPatriciaHashed(accountKeyLen int, ctx PatriciaContext, tmpdir string) *HexPatriciaHashed { hph := &HexPatriciaHashed{ ctx: ctx, keccak: sha3.NewLegacyKeccak256().(keccakState), @@ -93,12 +93,7 @@ func NewHexPatriciaHashed(accountKeyLen int, ctx PatriciaContext) *HexPatriciaHa accountKeyLen: accountKeyLen, auxBuffer: bytes.NewBuffer(make([]byte, 8192)), } - tdir := os.TempDir() - if ctx != nil { - tdir = ctx.TempDir() - } - tdir = filepath.Join(tdir, "branch-encoder") - hph.branchEncoder = NewBranchEncoder(1024, tdir) + hph.branchEncoder = NewBranchEncoder(1024, filepath.Join(tmpdir, "branch-encoder")) return hph } diff --git a/erigon-lib/commitment/hex_patricia_hashed_bench_test.go b/erigon-lib/commitment/hex_patricia_hashed_bench_test.go index 643a6c1accd..23eeff96615 100644 --- a/erigon-lib/commitment/hex_patricia_hashed_bench_test.go +++ b/erigon-lib/commitment/hex_patricia_hashed_bench_test.go @@ -12,7 +12,7 @@ import ( func Benchmark_HexPatriciaHashed_ReviewKeys(b *testing.B) { ms := NewMockState(&testing.T{}) ctx := context.Background() - hph := NewHexPatriciaHashed(length.Addr, ms) + hph := NewHexPatriciaHashed(length.Addr, ms, ms.TempDir()) hph.SetTrace(false) builder := NewUpdateBuilder() diff --git a/erigon-lib/commitment/hex_patricia_hashed_fuzz_test.go b/erigon-lib/commitment/hex_patricia_hashed_fuzz_test.go index 7a0b5ae67d3..d53b0a5012f 100644 --- a/erigon-lib/commitment/hex_patricia_hashed_fuzz_test.go +++ b/erigon-lib/commitment/hex_patricia_hashed_fuzz_test.go @@ -36,8 +36,8 @@ func Fuzz_ProcessUpdate(f *testing.F) { ms := NewMockState(t) ms2 := NewMockState(t) - hph := NewHexPatriciaHashed(20, ms) - hphAnother := NewHexPatriciaHashed(20, ms2) + hph := NewHexPatriciaHashed(20, ms, ms.TempDir()) + hphAnother := NewHexPatriciaHashed(20, ms2, ms2.TempDir()) hph.SetTrace(false) hphAnother.SetTrace(false) @@ -142,8 +142,8 @@ func Fuzz_ProcessUpdates_ArbitraryUpdateCount(f *testing.F) { ms := NewMockState(t) ms2 := NewMockState(t) - hph := NewHexPatriciaHashed(20, ms) - hphAnother := NewHexPatriciaHashed(20, ms2) + hph := NewHexPatriciaHashed(20, ms, ms.TempDir()) + hphAnother := NewHexPatriciaHashed(20, ms2, ms2.TempDir()) plainKeys, updates := builder.Build() @@ -199,7 +199,7 @@ func Fuzz_HexPatriciaHashed_ReviewKeys(f *testing.F) { } ms := NewMockState(t) - hph := NewHexPatriciaHashed(length.Addr, ms) + hph := NewHexPatriciaHashed(length.Addr, ms, ms.TempDir()) hph.SetTrace(false) diff --git a/erigon-lib/commitment/hex_patricia_hashed_test.go b/erigon-lib/commitment/hex_patricia_hashed_test.go index 93741adb1f5..bb1ddb26731 100644 --- a/erigon-lib/commitment/hex_patricia_hashed_test.go +++ b/erigon-lib/commitment/hex_patricia_hashed_test.go @@ -36,7 +36,7 @@ import ( func Test_HexPatriciaHashed_ResetThenSingularUpdates(t *testing.T) { ctx := context.Background() ms := NewMockState(t) - hph := NewHexPatriciaHashed(1, ms) + hph := NewHexPatriciaHashed(1, ms, ms.TempDir()) hph.SetTrace(false) plainKeys, updates := NewUpdateBuilder(). Balance("00", 4). @@ -107,7 +107,7 @@ func Test_HexPatriciaHashed_ResetThenSingularUpdates(t *testing.T) { func Test_HexPatriciaHashed_EmptyUpdate(t *testing.T) { ms := NewMockState(t) ctx := context.Background() - hph := NewHexPatriciaHashed(1, ms) + hph := NewHexPatriciaHashed(1, ms, ms.TempDir()) hph.SetTrace(false) plainKeys, updates := NewUpdateBuilder(). Balance("00", 4). @@ -161,8 +161,8 @@ func Test_HexPatriciaHashed_UniqueRepresentation2(t *testing.T) { //Balance("0000000000000000000000000000000000000000", 4000000000000138901). Build() - trieOne := NewHexPatriciaHashed(20, ms) - trieTwo := NewHexPatriciaHashed(20, ms2) + trieOne := NewHexPatriciaHashed(20, ms, ms.TempDir()) + trieTwo := NewHexPatriciaHashed(20, ms2, ms2.TempDir()) //trieOne.SetTrace(true) //trieTwo.SetTrace(true) @@ -298,8 +298,8 @@ func Test_HexPatriciaHashed_BrokenUniqueRepr(t *testing.T) { Build() keyLen := 20 - trieSequential := NewHexPatriciaHashed(keyLen, stateSeq) - trieBatch := NewHexPatriciaHashed(keyLen, stateBatch) + trieSequential := NewHexPatriciaHashed(keyLen, stateSeq, stateSeq.TempDir()) + trieBatch := NewHexPatriciaHashed(keyLen, stateBatch, stateBatch.TempDir()) if sortHashedKeys { plainKeys, updates = sortUpdatesByHashIncrease(t, trieSequential, plainKeys, updates) @@ -387,8 +387,8 @@ func Test_HexPatriciaHashed_UniqueRepresentation(t *testing.T) { Storage("68ee6c0e9cdc73b2b2d52dbd79f19d24fe25e2f9", "d1664244ae1a8a05f8f1d41e45548fbb7aa54609b985d6439ee5fd9bb0da619f", "9898"). Build() - trieSequential := NewHexPatriciaHashed(length.Addr, stateSeq) - trieBatch := NewHexPatriciaHashed(length.Addr, stateBatch) + trieSequential := NewHexPatriciaHashed(length.Addr, stateSeq, stateSeq.TempDir()) + trieBatch := NewHexPatriciaHashed(length.Addr, stateBatch, stateBatch.TempDir()) plainKeys, updates = sortUpdatesByHashIncrease(t, trieSequential, plainKeys, updates) @@ -477,7 +477,7 @@ func Test_HexPatriciaHashed_Sepolia(t *testing.T) { }, } - hph := NewHexPatriciaHashed(length.Addr, ms) + hph := NewHexPatriciaHashed(length.Addr, ms, ms.TempDir()) //hph.SetTrace(true) for _, testData := range tests { @@ -611,8 +611,8 @@ func Test_HexPatriciaHashed_StateEncodeDecodeSetup(t *testing.T) { Storage("f5", "04", "9898"). Build() - before := NewHexPatriciaHashed(1, ms) - after := NewHexPatriciaHashed(1, ms) + before := NewHexPatriciaHashed(1, ms, ms.TempDir()) + after := NewHexPatriciaHashed(1, ms, ms.TempDir()) err := ms.applyPlainUpdates(plainKeys, updates) require.NoError(t, err) @@ -659,7 +659,7 @@ func Test_HexPatriciaHashed_StateRestoreAndContinue(t *testing.T) { Balance("ff", 900234). Build() - trieOne := NewHexPatriciaHashed(1, ms) + trieOne := NewHexPatriciaHashed(1, ms, ms.TempDir()) err := ms.applyPlainUpdates(plainKeys, updates) require.NoError(t, err) err = ms2.applyPlainUpdates(plainKeys, updates) @@ -681,7 +681,7 @@ func Test_HexPatriciaHashed_StateRestoreAndContinue(t *testing.T) { require.NotEmpty(t, buf) t.Logf("restore state to another trie\n") - trieTwo := NewHexPatriciaHashed(1, ms2) + trieTwo := NewHexPatriciaHashed(1, ms2, ms2.TempDir()) err = trieTwo.SetState(buf) require.NoError(t, err) @@ -744,8 +744,8 @@ func Test_HexPatriciaHashed_RestoreAndContinue(t *testing.T) { Storage("f5", "04", "9898"). Build() - trieOne := NewHexPatriciaHashed(1, ms) - trieTwo := NewHexPatriciaHashed(1, ms) + trieOne := NewHexPatriciaHashed(1, ms, ms.TempDir()) + trieTwo := NewHexPatriciaHashed(1, ms, ms.TempDir()) err := ms.applyPlainUpdates(plainKeys, updates) require.NoError(t, err) @@ -796,8 +796,8 @@ func Test_HexPatriciaHashed_ProcessUpdates_UniqueRepresentation_AfterStateRestor Storage("68ee6c0e9cdc73b2b2d52dbd79f19d24fe25e2f9", "d1664244ae1a8a05f8f1d41e45548fbb7aa54609b985d6439ee5fd9bb0da619f", "9898"). Build() - sequential := NewHexPatriciaHashed(20, seqState) - batch := NewHexPatriciaHashed(20, batchState) + sequential := NewHexPatriciaHashed(20, seqState, seqState.TempDir()) + batch := NewHexPatriciaHashed(20, batchState, batchState.TempDir()) plainKeys, updates = sortUpdatesByHashIncrease(t, sequential, plainKeys, updates) @@ -826,7 +826,7 @@ func Test_HexPatriciaHashed_ProcessUpdates_UniqueRepresentation_AfterStateRestor require.NoError(t, err) sequential.Reset() - sequential = NewHexPatriciaHashed(20, seqState) + sequential = NewHexPatriciaHashed(20, seqState, seqState.TempDir()) err = sequential.SetState(prevState) require.NoError(t, err) @@ -895,8 +895,8 @@ func Test_HexPatriciaHashed_ProcessUpdates_UniqueRepresentationInTheMiddle(t *te Balance("ba7a3b7b095d3370c022ca655c790f0c0ead66f5", 5*1e17). Build() - sequential := NewHexPatriciaHashed(20, seqState) - batch := NewHexPatriciaHashed(20, batchState) + sequential := NewHexPatriciaHashed(20, seqState, seqState.TempDir()) + batch := NewHexPatriciaHashed(20, batchState, batchState.TempDir()) plainKeys, updates = sortUpdatesByHashIncrease(t, sequential, plainKeys, updates) @@ -927,7 +927,7 @@ func Test_HexPatriciaHashed_ProcessUpdates_UniqueRepresentationInTheMiddle(t *te require.NoError(t, err) sequential.Reset() - sequential = NewHexPatriciaHashed(20, seqState) + sequential = NewHexPatriciaHashed(20, seqState, seqState.TempDir()) err = sequential.SetState(prevState) require.NoError(t, err)