From b014d2cd55a977e71e529704ce15eecc40423dc9 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Tue, 14 Jan 2025 09:58:01 +0100 Subject: [PATCH] updates --- v2/cmd/gen/gen.go | 26 +++++++++++--------------- v2/cmd/rollback/rollback.go | 12 +++--------- v2/cmd/snapshot/snapshot.go | 12 +++--------- v2/go.mod | 6 +++--- v2/go.sum | 16 +++++++--------- v2/logger.go | 21 +++++++++++++++++++++ v2/multitree.go | 17 +++++++++-------- v2/tree_test.go | 16 ++++++++-------- 8 files changed, 65 insertions(+), 61 deletions(-) diff --git a/v2/cmd/gen/gen.go b/v2/cmd/gen/gen.go index 9eb8631ec..a9872e63e 100644 --- a/v2/cmd/gen/gen.go +++ b/v2/cmd/gen/gen.go @@ -12,15 +12,10 @@ import ( "github.com/dustin/go-humanize" "github.com/kocubinski/costor-api/compact" "github.com/kocubinski/costor-api/core" - "github.com/rs/zerolog" - zlog "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) -var log = zlog.Output(zerolog.ConsoleWriter{ - Out: os.Stderr, - TimeFormat: time.Stamp, -}) +var log = iavl.NewTestLogger() func Command() *cobra.Command { cmd := &cobra.Command{ @@ -75,9 +70,10 @@ func emitCommand() *cobra.Command { go func() { stats, err := stream.Compact() if err != nil { - log.Fatal().Err(err).Msg("failed to compact") + log.Error("failed to compact", "error", err) + os.Exit(1) } - log.Info().Msgf(stats.Report()) + log.Info(stats.Report()) wg.Done() }() @@ -95,13 +91,13 @@ func emitCommand() *cobra.Command { if itr.Version() < int64(start) { if cnt%5_000_000 == 0 { - log.Info().Msgf("fast forward version=%d nodes=%s", itr.Version(), humanize.Comma(cnt)) + log.Info(fmt.Sprintf("fast forward version=%d nodes=%s", itr.Version(), humanize.Comma(cnt))) } continue } if cnt%500_000 == 0 { - log.Info().Msgf("version=%d nodes=%s", itr.Version(), humanize.Comma(cnt)) + log.Info(fmt.Sprintf("version=%d nodes=%s", itr.Version(), humanize.Comma(cnt))) } select { @@ -149,11 +145,11 @@ func treeCommand() *cobra.Command { Use: "tree", Short: "build and save a Tree to disk, taking generated changesets as input", RunE: func(cmd *cobra.Command, args []string) error { - multiTree := iavl.NewMultiTree(dbPath, iavl.TreeOptions{StateStorage: true}) + multiTree := iavl.NewMultiTree(iavl.NewTestLogger(), dbPath, iavl.TreeOptions{StateStorage: true}) defer func(mt *iavl.MultiTree) { err := mt.Close() if err != nil { - log.Error().Err(err).Msg("failed to close db") + log.Error("failed to close db", "error", err) } }(multiTree) @@ -203,12 +199,12 @@ func treeCommand() *cobra.Command { i++ if i%100_000 == 0 { - log.Info().Msgf("leaves=%s dur=%s rate=%s version=%d", + log.Info(fmt.Sprintf("leaves=%s dur=%s rate=%s version=%d", humanize.Comma(i), time.Since(start), humanize.Comma(int64(100_000/time.Since(start).Seconds())), itr.Version(), - ) + )) start = time.Now() } } @@ -219,7 +215,7 @@ func treeCommand() *cobra.Command { } } - log.Info().Msgf("last version=%d hash=%x", lastVersion, lastHash) + log.Info(fmt.Sprintf("last version=%d hash=%x", lastVersion, lastHash)) return nil }, diff --git a/v2/cmd/rollback/rollback.go b/v2/cmd/rollback/rollback.go index 1a76ad2f9..3688e28be 100644 --- a/v2/cmd/rollback/rollback.go +++ b/v2/cmd/rollback/rollback.go @@ -1,19 +1,13 @@ package rollback import ( - "os" - "time" + "fmt" "github.com/cosmos/iavl/v2" - "github.com/rs/zerolog" - zlog "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) -var log = zlog.Output(zerolog.ConsoleWriter{ - Out: os.Stderr, - TimeFormat: time.Stamp, -}) +var log = iavl.NewTestLogger() func Command() *cobra.Command { var ( @@ -29,7 +23,7 @@ func Command() *cobra.Command { return err } for _, dbPath := range dbPaths { - log.Info().Msgf("revert db %s to version %d", dbPath, version) + log.Info(fmt.Sprintf("revert db %s to version %d", dbPath, version)) sql, err := iavl.NewSqliteDb(iavl.NewNodePool(), iavl.SqliteDbOptions{Path: dbPath}) if err != nil { return err diff --git a/v2/cmd/snapshot/snapshot.go b/v2/cmd/snapshot/snapshot.go index 27e10b64f..6e1976618 100644 --- a/v2/cmd/snapshot/snapshot.go +++ b/v2/cmd/snapshot/snapshot.go @@ -1,19 +1,13 @@ package snapshot import ( - "os" - "time" + "fmt" "github.com/cosmos/iavl/v2" - "github.com/rs/zerolog" - zlog "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) -var log = zlog.Output(zerolog.ConsoleWriter{ - Out: os.Stderr, - TimeFormat: time.Stamp, -}) +var log = iavl.NewTestLogger() func Command() *cobra.Command { var ( @@ -28,7 +22,7 @@ func Command() *cobra.Command { if err != nil { return err } - log.Info().Msgf("found db paths: %v", paths) + log.Info(fmt.Sprintf("found db paths: %v", paths)) var ( pool = iavl.NewNodePool() diff --git a/v2/go.mod b/v2/go.mod index b08f3ec20..ba7bc4841 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -10,7 +10,6 @@ require ( github.com/emicklei/dot v1.6.0 github.com/kocubinski/costor-api v1.1.1 github.com/prometheus/client_golang v1.16.0 - github.com/rs/zerolog v1.30.0 github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.4 golang.org/x/exp v0.0.0-20231006140011-7918f672742d @@ -23,14 +22,15 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.10.1 // indirect + github.com/rs/zerolog v1.33.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - golang.org/x/sys v0.13.0 // indirect + golang.org/x/sys v0.22.0 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/v2/go.sum b/v2/go.sum index 9a008cd50..f081ddc81 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -30,13 +30,12 @@ github.com/kocubinski/costor-api v1.1.1 h1:sgfJA7T/8IfZ59zxiMrED0xdjerAFuPNBTqyO github.com/kocubinski/costor-api v1.1.1/go.mod h1:ESMBMDkKfN+9vvvhhNVdKLhbOmzI3O/i16iXvRM9Tuc= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -52,8 +51,8 @@ github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+Pymzi github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c= -github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w= +github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= +github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= @@ -64,12 +63,11 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= diff --git a/v2/logger.go b/v2/logger.go index 2a66a7752..2bfe1a401 100644 --- a/v2/logger.go +++ b/v2/logger.go @@ -1,5 +1,7 @@ package iavl +import "log/slog" + // Logger defines basic logger that IAVL expects. // It is a subset of the cosmossdk.io/core/log.Logger interface. // cosmossdk.io/log/log.Logger implements this interface. @@ -32,3 +34,22 @@ func (l *noopLogger) Info(string, ...any) {} func (l *noopLogger) Warn(string, ...any) {} func (l *noopLogger) Error(string, ...any) {} func (l *noopLogger) Debug(string, ...any) {} + +func NewTestLogger() Logger { + return &testLogger{} +} + +type testLogger struct{} + +func (l *testLogger) Info(msg string, keys ...any) { + slog.Info(msg, keys...) +} +func (l *testLogger) Warn(msg string, keys ...any) { + slog.Warn(msg, keys...) +} +func (l *testLogger) Error(msg string, keys ...any) { + slog.Error(msg, keys...) +} +func (l *testLogger) Debug(msg string, keys ...any) { + slog.Debug(msg, keys...) +} diff --git a/v2/multitree.go b/v2/multitree.go index c6d809588..fa46e1933 100644 --- a/v2/multitree.go +++ b/v2/multitree.go @@ -9,13 +9,14 @@ import ( "github.com/cosmos/iavl/v2/metrics" "github.com/dustin/go-humanize" - "github.com/rs/zerolog/log" "golang.org/x/exp/slices" ) // MultiTree encapsulates multiple IAVL trees, each with its own "store key" in the context of the Cosmos SDK. // Within IAVL v2 is only used to test the IAVL v2 implementation, and for import/export of IAVL v2 state. type MultiTree struct { + logger Logger + Trees map[string]*Tree pool *NodePool @@ -27,7 +28,7 @@ type MultiTree struct { errorCh chan error } -func NewMultiTree(rootPath string, opts TreeOptions) *MultiTree { +func NewMultiTree(logger Logger, rootPath string, opts TreeOptions) *MultiTree { return &MultiTree{ Trees: make(map[string]*Tree), doneCh: make(chan saveVersionResult, 1000), @@ -38,8 +39,8 @@ func NewMultiTree(rootPath string, opts TreeOptions) *MultiTree { } } -func ImportMultiTree(pool *NodePool, version int64, path string, treeOpts TreeOptions) (*MultiTree, error) { - mt := NewMultiTree(path, treeOpts) +func ImportMultiTree(logger Logger, pool *NodePool, version int64, path string, treeOpts TreeOptions) (*MultiTree, error) { + mt := NewMultiTree(logger, path, treeOpts) paths, err := FindDbsInPath(path) if err != nil { return nil, err @@ -79,7 +80,7 @@ func ImportMultiTree(pool *NodePool, version int64, path string, treeOpts TreeOp return nil, err case res := <-done: prefix := filepath.Base(res.path) - log.Info().Msgf("imported %s", prefix) + logger.Info(fmt.Sprintf("imported %s", prefix)) mt.Trees[prefix] = res.tree } } @@ -173,7 +174,7 @@ func (mt *MultiTree) SaveVersionConcurrently() ([]byte, int64, error) { for i := 0; i < treeCount; i++ { select { case err := <-mt.errorCh: - log.Error().Err(err).Msg("failed to save version") + mt.logger.Error("failed to save version", "error", err) errs = append(errs, err) case result := <-mt.doneCh: if version != -1 && version != result.version { @@ -218,7 +219,7 @@ func (mt *MultiTree) SnapshotConcurrently() error { for i := 0; i < treeCount; i++ { select { case err := <-mt.errorCh: - log.Error().Err(err).Msg("failed to snapshot") + mt.logger.Error("failed to snapshot", "error", err) errs = append(errs, err) case <-mt.doneCh: } @@ -271,7 +272,7 @@ func (mt *MultiTree) WarmLeaves() error { for i := 0; i < cnt; i++ { select { case err := <-mt.errorCh: - log.Error().Err(err).Msg("failed to warm leaves") + mt.logger.Error("failed to warm leaves", "error", err) return err case <-mt.doneCh: } diff --git a/v2/tree_test.go b/v2/tree_test.go index 7bd5aa53a..2083722f7 100644 --- a/v2/tree_test.go +++ b/v2/tree_test.go @@ -172,7 +172,7 @@ func TestTree_Hash(t *testing.T) { treeOpts := TreeOptions{CheckpointInterval: 10, HeightFilter: 1, StateStorage: true, EvictionDepth: 8} testStart := time.Now() - multiTree := NewMultiTree(tmpDir, treeOpts) + multiTree := NewMultiTree(NewTestLogger(), tmpDir, treeOpts) itrs, ok := opts.Iterator.(*bench.ChangesetIterators) require.True(t, ok) for _, sk := range itrs.StoreKeys() { @@ -190,7 +190,7 @@ func TestTree_Build_Load(t *testing.T) { //tmpDir := t.TempDir() tmpDir := "/tmp/iavl-v2-test" opts := testutil.NewTreeBuildOptions().With10_000() - multiTree := NewMultiTree(tmpDir, TreeOptions{CheckpointInterval: 4000, HeightFilter: 0, StateStorage: false}) + multiTree := NewMultiTree(NewTestLogger(), tmpDir, TreeOptions{CheckpointInterval: 4000, HeightFilter: 0, StateStorage: false}) itrs, ok := opts.Iterator.(*bench.ChangesetIterators) require.True(t, ok) for _, sk := range itrs.StoreKeys() { @@ -205,7 +205,7 @@ func TestTree_Build_Load(t *testing.T) { require.NoError(t, multiTree.Close()) t.Log("import snapshot into new tree") - mt, err := ImportMultiTree(multiTree.pool, 10_000, tmpDir, DefaultTreeOptions()) + mt, err := ImportMultiTree(NewTestLogger(), multiTree.pool, 10_000, tmpDir, DefaultTreeOptions()) require.NoError(t, err) t.Log("build tree to version 12,000 and verify hash") @@ -218,7 +218,7 @@ func TestTree_Build_Load(t *testing.T) { t.Log("export the tree at version 12,000 and import it into a sql db in pre-order") traverseOrder := PreOrder - restorePreOrderMt := NewMultiTree(t.TempDir(), TreeOptions{CheckpointInterval: 4000}) + restorePreOrderMt := NewMultiTree(NewTestLogger(), t.TempDir(), TreeOptions{CheckpointInterval: 4000}) for sk, tree := range multiTree.Trees { require.NoError(t, restorePreOrderMt.MountTree(sk)) exporter := tree.Export(traverseOrder) @@ -232,7 +232,7 @@ func TestTree_Build_Load(t *testing.T) { t.Log("export the tree at version 12,000 and import it into a sql db in post-order") traverseOrder = PostOrder - restorePostOrderMt := NewMultiTree(t.TempDir(), TreeOptions{CheckpointInterval: 4000}) + restorePostOrderMt := NewMultiTree(NewTestLogger(), t.TempDir(), TreeOptions{CheckpointInterval: 4000}) for sk, tree := range multiTree.Trees { require.NoError(t, restorePostOrderMt.MountTree(sk)) exporter := tree.Export(traverseOrder) @@ -262,7 +262,7 @@ func TestOsmoLike_HotStart(t *testing.T) { // logDir := "/tmp/osmo-like-many-v2" logDir := "/Users/mattk/src/scratch/osmo-like-many/v2" pool := NewNodePool() - multiTree, err := ImportMultiTree(pool, 1, tmpDir, TreeOptions{HeightFilter: 0, StateStorage: false}) + multiTree, err := ImportMultiTree(NewTestLogger(), pool, 1, tmpDir, TreeOptions{HeightFilter: 0, StateStorage: false}) require.NoError(t, err) require.NotNil(t, multiTree) opts := testutil.CompactedChangelogs(logDir) @@ -284,7 +284,7 @@ func TestOsmoLike_ColdStart(t *testing.T) { treeOpts.HeightFilter = 1 treeOpts.EvictionDepth = 16 treeOpts.MetricsProxy = newPrometheusMetricsProxy() - multiTree := NewMultiTree(tmpDir, treeOpts) + multiTree := NewMultiTree(NewTestLogger(), tmpDir, treeOpts) require.NoError(t, multiTree.MountTrees()) require.NoError(t, multiTree.LoadVersion(1)) // require.NoError(t, multiTree.WarmLeaves()) @@ -627,7 +627,7 @@ func Test_Prune_Logic(t *testing.T) { func Test_Prune_Performance(t *testing.T) { tmpDir := "/tmp/iavl-v2" - multiTree := NewMultiTree(tmpDir, TreeOptions{CheckpointInterval: 50, StateStorage: false}) + multiTree := NewMultiTree(NewTestLogger(), tmpDir, TreeOptions{CheckpointInterval: 50, StateStorage: false}) require.NoError(t, multiTree.MountTrees()) require.NoError(t, multiTree.LoadVersion(1)) require.NoError(t, multiTree.WarmLeaves())