Skip to content

Commit

Permalink
Merge pull request #2195 from CortexFoundation/dev
Browse files Browse the repository at this point in the history
break rawdb -> {leveldb, pebble} dependency
  • Loading branch information
ucwong authored Nov 7, 2024
2 parents 5bc7990 + d669a74 commit e5466a2
Show file tree
Hide file tree
Showing 6 changed files with 137 additions and 119 deletions.
30 changes: 15 additions & 15 deletions core/bench_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"github.com/CortexFoundation/CortexTheseus/core/vm"
"github.com/CortexFoundation/CortexTheseus/crypto"
"github.com/CortexFoundation/CortexTheseus/ctxcdb"
"github.com/CortexFoundation/CortexTheseus/ctxcdb/pebble"
"github.com/CortexFoundation/CortexTheseus/params"
)

Expand Down Expand Up @@ -150,18 +151,13 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) {
if !disk {
db = rawdb.NewMemoryDatabase()
} else {
dir, err := os.MkdirTemp("", "eth-core-bench")
if err != nil {
b.Fatalf("cannot create temporary directory: %v", err)
}
defer os.RemoveAll(dir)
db, err = rawdb.NewLevelDBDatabase(dir, 128, 128, "", false)
pdb, err := pebble.New(b.TempDir(), 128, 128, "", false)
if err != nil {
b.Fatalf("cannot create temporary database: %v", err)
}
db = rawdb.NewDatabase(pdb)
defer db.Close()
}

// Generate a chain of b.N blocks using the supplied block
// generator function.
gspec := Genesis{
Expand Down Expand Up @@ -249,14 +245,15 @@ func makeChainForBench(db ctxcdb.Database, full bool, count uint64) {

func benchWriteChain(b *testing.B, full bool, count uint64) {
for i := 0; i < b.N; i++ {
dir, err := os.MkdirTemp("", "eth-chain-bench")
dir, err := os.MkdirTemp("", "ctxc-chain-bench")
if err != nil {
b.Fatalf("cannot create temporary directory: %v", err)
}
db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false)
pdb, err := pebble.New(b.TempDir(), 1024, 128, "", false)
if err != nil {
b.Fatalf("error opening database at %v: %v", dir, err)
b.Fatalf("error opening database: %v", err)
}
db := rawdb.NewDatabase(pdb)
makeChainForBench(db, full, count)
db.Close()
os.RemoveAll(dir)
Expand All @@ -270,26 +267,29 @@ func benchReadChain(b *testing.B, full bool, count uint64) {
}
defer os.RemoveAll(dir)

db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false)
pdb, err := pebble.New(dir, 1024, 128, "", false)
if err != nil {
b.Fatalf("error opening database at %v: %v", dir, err)
b.Fatalf("error opening database: %v", err)
}
db := rawdb.NewDatabase(pdb)

makeChainForBench(db, full, count)
db.Close()

b.ReportAllocs()
b.ResetTimer()

for i := 0; i < b.N; i++ {
db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false)
pdb, err = pebble.New(dir, 1024, 128, "", false)
if err != nil {
b.Fatalf("error opening database at %v: %v", dir, err)
b.Fatalf("error opening database: %v", err)
}
db = rawdb.NewDatabase(pdb)
chain, err := NewBlockChain(db, nil, params.TestChainConfig, &CuckooFakeForTest{}, vm.Config{}, nil, nil)

if err != nil {
b.Fatalf("error creating chain: %v", err)
}

for n := uint64(0); n < count; n++ {
header := chain.GetHeaderByNumber(n)
if full {
Expand Down
94 changes: 4 additions & 90 deletions core/rawdb/database.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,7 @@ import (

"github.com/CortexFoundation/CortexTheseus/common"
"github.com/CortexFoundation/CortexTheseus/ctxcdb"
"github.com/CortexFoundation/CortexTheseus/ctxcdb/leveldb"
"github.com/CortexFoundation/CortexTheseus/ctxcdb/memorydb"
"github.com/CortexFoundation/CortexTheseus/ctxcdb/pebble"
"github.com/CortexFoundation/CortexTheseus/log"
)

Expand Down Expand Up @@ -308,30 +306,9 @@ func NewMemoryDatabaseWithCap(size int) ctxcdb.Database {
return NewDatabase(memorydb.NewWithCap(size))
}

// NewLevelDBDatabase creates a persistent key-value database without a freezer
// moving immutable chain segments into cold storage.
func NewLevelDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ctxcdb.Database, error) {
db, err := leveldb.New(file, cache, handles, namespace, readonly)
if err != nil {
return nil, err
}
log.Info("Using LevelDB as the backing database")
return NewDatabase(db), nil
}

// NewPebbleDBDatabase creates a persistent key-value database without a freezer
// moving immutable chain segments into cold storage.
func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ctxcdb.Database, error) {
db, err := pebble.New(file, cache, handles, namespace, readonly)
if err != nil {
return nil, err
}
return NewDatabase(db), nil
}

const (
dbPebble = "pebble"
dbLeveldb = "leveldb"
DBPebble = "pebble"
DBLeveldb = "leveldb"
)

// PreexistingDatabase checks the given data directory whether a database is already
Expand All @@ -345,72 +322,9 @@ func PreexistingDatabase(path string) string {
if err != nil {
panic(err) // only possible if the pattern is malformed
}
return dbPebble
}
return dbLeveldb
}

// OpenOptions contains the options to apply when opening a database.
// OBS: If AncientsDirectory is empty, it indicates that no freezer is to be used.
type OpenOptions struct {
Type string // "leveldb" | "pebble"
Directory string // the datadir
AncientsDirectory string // the ancients-dir
Namespace string // the namespace for database relevant metrics
Cache int // the capacity(in megabytes) of the data caching
Handles int // number of files to be open simultaneously
ReadOnly bool
}

// openKeyValueDatabase opens a disk-based key-value database, e.g. leveldb or pebble.
//
// type == null type != null
// +----------------------------------------
// db is non-existent | pebble default | specified type
// db is existent | from db | specified type (if compatible)
func openKeyValueDatabase(o OpenOptions) (ctxcdb.Database, error) {
// Reject any unsupported database type
if len(o.Type) != 0 && o.Type != dbLeveldb && o.Type != dbPebble {
return nil, fmt.Errorf("unknown db.engine %v", o.Type)
}
// Retrieve any pre-existing database's type and use that or the requested one
// as long as there's no conflict between the two types
existingDb := PreexistingDatabase(o.Directory)
if len(existingDb) != 0 && len(o.Type) != 0 && o.Type != existingDb {
return nil, fmt.Errorf("db.engine choice was %v but found pre-existing %v database in specified data directory", o.Type, existingDb)
}
if o.Type == dbPebble || existingDb == dbPebble {
log.Info("Using pebble as the backing database")
return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
}
if o.Type == dbLeveldb || existingDb == dbLeveldb {
log.Info("Using leveldb as the backing database")
return NewLevelDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
}
// No pre-existing database, no user-requested one either. Default to Pebble.
log.Info("Defaulting to pebble as the backing database")
return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
}

// Open opens both a disk-based key-value database such as leveldb or pebble, but also
// integrates it with a freezer database -- if the AncientDir option has been
// set on the provided OpenOptions.
// The passed o.AncientDir indicates the path of root ancient directory where
// the chain freezer can be opened.
func Open(o OpenOptions) (ctxcdb.Database, error) {
kvdb, err := openKeyValueDatabase(o)
if err != nil {
return nil, err
}
if len(o.AncientsDirectory) == 0 {
return kvdb, nil
}
frdb, err := NewDatabaseWithFreezer(kvdb, o.AncientsDirectory, o.Namespace, o.ReadOnly)
if err != nil {
kvdb.Close()
return nil, err
return DBPebble
}
return frdb, nil
return DBLeveldb
}

type counter uint64
Expand Down
12 changes: 3 additions & 9 deletions ctxc/filters/bench_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,10 +66,7 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) {
benchDataDir := node.DefaultDataDir() + "/cortex/chaindata"
b.Log("Running bloombits benchmark section size:", sectionSize)

db, err := rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false)
if err != nil {
b.Fatalf("error opening database at %v: %v", benchDataDir, err)
}
var db = rawdb.NewMemoryDatabase()
head := rawdb.ReadHeadBlockHash(db)
if head == (common.Hash{}) {
b.Fatalf("chain data not found at %v", benchDataDir)
Expand Down Expand Up @@ -130,7 +127,7 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) {
for i := 0; i < benchFilterCnt; i++ {
if i%20 == 0 {
db.Close()
db, _ = rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false)
db = rawdb.NewMemoryDatabase()
backend = &testBackend{db: db, sections: cnt}
sys = NewFilterSystem(backend, Config{})
}
Expand Down Expand Up @@ -164,10 +161,7 @@ func BenchmarkNoBloomBits(b *testing.B) {
b.Skip("test disabled: this tests presume (and modify) an existing datadir.")
benchDataDir := node.DefaultDataDir() + "/cortex/chaindata"
b.Log("Running benchmark without bloombits")
db, err := rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false)
if err != nil {
b.Fatalf("error opening database at %v: %v", benchDataDir, err)
}
var db = rawdb.NewMemoryDatabase()
head := rawdb.ReadHeadBlockHash(db)
if head == (common.Hash{}) {
b.Fatalf("chain data not found at %v", benchDataDir)
Expand Down
4 changes: 2 additions & 2 deletions ctxc/filters/filter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ func BenchmarkFilters(b *testing.B) {
defer os.RemoveAll(dir)

var (
db, _ = rawdb.NewLevelDBDatabase(dir, 0, 0, "", false)
db = rawdb.NewMemoryDatabase()
//backend = &testBackend{db: db}
key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
addr1 = crypto.PubkeyToAddress(key1.PublicKey)
Expand Down Expand Up @@ -104,7 +104,7 @@ func TestFilters(t *testing.T) {
defer os.RemoveAll(dir)

var (
db, _ = rawdb.NewLevelDBDatabase(dir, 0, 0, "", false)
db = rawdb.NewMemoryDatabase()
//backend = &testBackend{db: db}
key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
addr = crypto.PubkeyToAddress(key1.PublicKey)
Expand Down
111 changes: 111 additions & 0 deletions node/database.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
// Copyright 2024 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

package node

import (
"fmt"

"github.com/CortexFoundation/CortexTheseus/core/rawdb"
"github.com/CortexFoundation/CortexTheseus/ctxcdb"
"github.com/CortexFoundation/CortexTheseus/ctxcdb/leveldb"
"github.com/CortexFoundation/CortexTheseus/ctxcdb/pebble"
"github.com/CortexFoundation/CortexTheseus/log"
)

// openOptions contains the options to apply when opening a database.
// OBS: If AncientsDirectory is empty, it indicates that no freezer is to be used.
type openOptions struct {
Type string // "leveldb" | "pebble"
Directory string // the datadir
AncientsDirectory string // the ancients-dir
Namespace string // the namespace for database relevant metrics
Cache int // the capacity(in megabytes) of the data caching
Handles int // number of files to be open simultaneously
ReadOnly bool
}

// openDatabase opens both a disk-based key-value database such as leveldb or pebble, but also
// integrates it with a freezer database -- if the AncientDir option has been
// set on the provided OpenOptions.
// The passed o.AncientDir indicates the path of root ancient directory where
// the chain freezer can be opened.
func openDatabase(o openOptions) (ctxcdb.Database, error) {
kvdb, err := openKeyValueDatabase(o)
if err != nil {
return nil, err
}
if len(o.AncientsDirectory) == 0 {
return kvdb, nil
}
frdb, err := rawdb.NewDatabaseWithFreezer(kvdb, o.AncientsDirectory, o.Namespace, o.ReadOnly)
if err != nil {
kvdb.Close()
return nil, err
}
return frdb, nil
}

// openKeyValueDatabase opens a disk-based key-value database, e.g. leveldb or pebble.
//
// type == null type != null
// +----------------------------------------
// db is non-existent | pebble default | specified type
// db is existent | from db | specified type (if compatible)
func openKeyValueDatabase(o openOptions) (ctxcdb.Database, error) {
// Reject any unsupported database type
if len(o.Type) != 0 && o.Type != rawdb.DBLeveldb && o.Type != rawdb.DBPebble {
return nil, fmt.Errorf("unknown db.engine %v", o.Type)
}
// Retrieve any pre-existing database's type and use that or the requested one
// as long as there's no conflict between the two types
existingDb := rawdb.PreexistingDatabase(o.Directory)
if len(existingDb) != 0 && len(o.Type) != 0 && o.Type != existingDb {
return nil, fmt.Errorf("db.engine choice was %v but found pre-existing %v database in specified data directory", o.Type, existingDb)
}
if o.Type == rawdb.DBPebble || existingDb == rawdb.DBPebble {
log.Info("Using pebble as the backing database")
return newPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
}
if o.Type == rawdb.DBLeveldb || existingDb == rawdb.DBLeveldb {
log.Info("Using leveldb as the backing database")
return newLevelDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
}
// No pre-existing database, no user-requested one either. Default to Pebble.
log.Info("Defaulting to pebble as the backing database")
return newPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
}

// newLevelDBDatabase creates a persistent key-value database without a freezer
// moving immutable chain segments into cold storage.
func newLevelDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ctxcdb.Database, error) {
db, err := leveldb.New(file, cache, handles, namespace, readonly)
if err != nil {
return nil, err
}
log.Info("Using LevelDB as the backing database")
return rawdb.NewDatabase(db), nil
}

// newPebbleDBDatabase creates a persistent key-value database without a freezer
// moving immutable chain segments into cold storage.
func newPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ctxcdb.Database, error) {
db, err := pebble.New(file, cache, handles, namespace, readonly)
if err != nil {
return nil, err
}
return rawdb.NewDatabase(db), nil
}
5 changes: 2 additions & 3 deletions node/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -705,7 +705,7 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, r
if n.config.DataDir == "" {
db = rawdb.NewMemoryDatabase()
} else {
db, err = rawdb.Open(rawdb.OpenOptions{
db, err = openDatabase(openOptions{
Type: n.config.DBEngine,
Directory: n.ResolvePath(name),
Namespace: namespace,
Expand All @@ -714,7 +714,6 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, r
ReadOnly: readonly,
})
}

if err == nil {
db = n.wrapDatabase(db)
}
Expand All @@ -739,7 +738,7 @@ func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, ancient
if n.config.DataDir == "" {
db, err = rawdb.NewDatabaseWithFreezer(memorydb.New(), "", namespace, readonly)
} else {
db, err = rawdb.Open(rawdb.OpenOptions{
db, err = openDatabase(openOptions{
Type: n.config.DBEngine,
Directory: n.ResolvePath(name),
AncientsDirectory: n.ResolveAncient(name, ancient),
Expand Down

0 comments on commit e5466a2

Please sign in to comment.