diff --git a/core/blockchain.go b/core/blockchain.go index 358cdcd30f41..4f8a2f1ade82 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -100,7 +100,23 @@ const ( triesInMemory = 128 // BlockChainVersion ensures that an incompatible database forces a resync from scratch. - BlockChainVersion = 3 + // + // Changelog: + // + // - Version 4 + // The following incompatible database changes were added: + // * the `BlockNumber`, `TxHash`, `TxIndex`, `BlockHash` and `Index` fields of log are deleted + // * the `Bloom` field of receipt is deleted + // * the `BlockIndex` and `TxIndex` fields of txlookup are deleted + // - Version 5 + // The following incompatible database changes were added: + // * the `TxHash`, `GasCost`, and `ContractAddress` fields are no longer stored for a receipt + // * the `TxHash`, `GasCost`, and `ContractAddress` fields are computed by looking up the + // receipts' corresponding block + // - Version 6 + // The following incompatible database changes were added: + // * Transaction lookup information stores the corresponding block number instead of block hash + BlockChainVersion uint64 = 6 // Maximum length of chain to cache by block's number blocksHashCacheLimit = 900 @@ -1220,7 +1236,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ } // Compute all the non-consensus fields of the receipts if err := receipts.DeriveFields(bc.chainConfig, blockHash, blockNumber, block.BaseFee(), block.Transactions()); err != nil { - return i, fmt.Errorf("failed to set receipts data: %v", err) + return i, fmt.Errorf("failed to derive receipts data: %v", err) } // Write all the data out into the database rawdb.WriteBody(batch, blockHash, blockNumber, block.Body()) diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index f968dcd76ef8..d6fbad50b5b5 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -476,18 +476,6 @@ func DeleteReceipts(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { } } -// storedReceiptRLP is the storage encoding of a receipt. -// Re-definition in core/types/receipt.go. -type storedReceiptRLP struct { - PostStateOrStatus []byte - CumulativeGasUsed uint64 - Bloom types.Bloom - TxHash common.Hash - ContractAddress common.Address - Logs []*types.LogForStorage - GasUsed uint64 -} - // ReceiptLogs is a barebone version of ReceiptForStorage which only keeps // the list of logs. When decoding a stored receipt into this object we // avoid creating the bloom filter. @@ -497,7 +485,7 @@ type receiptLogs struct { // DecodeRLP implements rlp.Decoder. func (r *receiptLogs) DecodeRLP(s *rlp.Stream) error { - var stored storedReceiptRLP + var stored types.ReceiptForStorage if err := s.Decode(&stored); err != nil { return err } diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index d3d2b0e84a77..8d45ed5d2c29 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -19,6 +19,7 @@ package rawdb import ( "bytes" "encoding/hex" + "fmt" "math/big" "os" "testing" @@ -339,22 +340,48 @@ func TestBlockReceiptStorage(t *testing.T) { if rs := ReadReceipts(db, hash, 0, params.TestChainConfig); len(rs) == 0 { t.Fatalf("no receipts returned") } else { - for i := 0; i < len(receipts); i++ { - rlpHave, _ := rlp.EncodeToBytes(rs[i]) - rlpWant, _ := rlp.EncodeToBytes(receipts[i]) - - if !bytes.Equal(rlpHave, rlpWant) { - t.Fatalf("receipt #%d: receipt mismatch: have %v, want %v", i, rs[i], receipts[i]) - } + if err := checkReceiptsRLP(rs, receipts); err != nil { + t.Fatalf("fail to checkReceiptsRLP %v", err) } } - // Delete the receipt slice and check purge + // Delete the body and ensure that the receipts are no longer returned (metadata can't be recomputed) + DeleteBody(db, hash, 0) + if rs := ReadReceipts(db, hash, 0, params.TestChainConfig); rs != nil { + t.Fatalf("receipts returned when body was deleted: %v", rs) + } + // Ensure that receipts without metadata can be returned without the block body too + if err := checkReceiptsRLP(ReadRawReceipts(db, hash, 0), receipts); err != nil { + t.Fatalf("fail to checkReceiptsRLP %v", err) + } + // Sanity check that body alone without the receipt is a full purge + WriteBody(db, hash, 0, body) + DeleteReceipts(db, hash, 0) if rs := ReadReceipts(db, hash, 0, params.TestChainConfig); len(rs) != 0 { t.Fatalf("deleted receipts returned: %v", rs) } } +func checkReceiptsRLP(have, want types.Receipts) error { + if len(have) != len(want) { + return fmt.Errorf("receipts sizes mismatch: have %d, want %d", len(have), len(want)) + } + for i := 0; i < len(want); i++ { + rlpHave, err := rlp.EncodeToBytes(have[i]) + if err != nil { + return err + } + rlpWant, err := rlp.EncodeToBytes(want[i]) + if err != nil { + return err + } + if !bytes.Equal(rlpHave, rlpWant) { + return fmt.Errorf("receipt #%d: receipt mismatch: have %s, want %s", i, hex.EncodeToString(rlpHave), hex.EncodeToString(rlpWant)) + } + } + return nil +} + // Tests that logs associated with a single block can be retrieved. func TestReadLogs(t *testing.T) { db := NewMemoryDatabase() diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index 21bdcf3d298f..7197fc5d52a5 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -17,6 +17,8 @@ package rawdb import ( + "math/big" + "github.com/XinFinOrg/XDPoSChain/common" "github.com/XinFinOrg/XDPoSChain/core/types" "github.com/XinFinOrg/XDPoSChain/ethdb" @@ -25,45 +27,37 @@ import ( "github.com/XinFinOrg/XDPoSChain/rlp" ) -type TxLookupEntry struct { - BlockHash common.Hash - BlockIndex uint64 - Index uint64 -} - // ReadTxLookupEntry retrieves the positional metadata associated with a transaction // hash to allow retrieving the transaction or receipt by hash. -func ReadTxLookupEntry(db ethdb.Reader, hash common.Hash) (common.Hash, uint64, uint64) { - // Load the positional metadata from disk and bail if it fails +func ReadTxLookupEntry(db ethdb.Reader, hash common.Hash) *uint64 { data, _ := db.Get(txLookupKey(hash)) if len(data) == 0 { - return common.Hash{}, 0, 0 + return nil + } + // Database v6 tx lookup just stores the block number + if len(data) < common.HashLength { + number := new(big.Int).SetBytes(data).Uint64() + return &number } - // Parse and return the contents of the lookup entry - var entry TxLookupEntry + // Database v4-v5 tx lookup format just stores the hash + if len(data) == common.HashLength { + return ReadHeaderNumber(db, common.BytesToHash(data)) + } + // Finally try database v3 tx lookup format + var entry LegacyTxLookupEntry if err := rlp.DecodeBytes(data, &entry); err != nil { - log.Error("Invalid lookup entry RLP", "hash", hash, "err", err) - return common.Hash{}, 0, 0 + log.Error("Invalid transaction lookup entry RLP", "hash", hash, "blob", data, "err", err) + return nil } - return entry.BlockHash, entry.BlockIndex, entry.Index + return &entry.BlockIndex } // WriteTxLookupEntriesByBlock stores a positional metadata for every transaction from // a block, enabling hash based transaction and receipt lookups. func WriteTxLookupEntriesByBlock(db ethdb.KeyValueWriter, block *types.Block) { - // Iterate over each transaction and encode its metadata - for i, tx := range block.Transactions() { - entry := TxLookupEntry{ - BlockHash: block.Hash(), - BlockIndex: block.NumberU64(), - Index: uint64(i), - } - data, err := rlp.EncodeToBytes(entry) - if err != nil { - log.Crit("Failed to RLP encode TxLookupEntry", "err", err) - } - if err := db.Put(txLookupKey(tx.Hash()), data); err != nil { - log.Crit("Failed to store tx lookup entry", "err", err) + for _, tx := range block.Transactions() { + if err := db.Put(txLookupKey(tx.Hash()), block.Number().Bytes()); err != nil { + log.Crit("Failed to store transaction lookup entry", "err", err) } } } @@ -76,63 +70,82 @@ func DeleteTxLookupEntry(db ethdb.KeyValueWriter, hash common.Hash) { // ReadTransaction retrieves a specific transaction from the database, along with // its added positional metadata. func ReadTransaction(db ethdb.Reader, hash common.Hash) (*types.Transaction, common.Hash, uint64, uint64) { - // Retrieve the lookup metadata and resolve the transaction from the body - blockHash, blockNumber, txIndex := ReadTxLookupEntry(db, hash) - - if blockHash != (common.Hash{}) { - body := ReadBody(db, blockHash, blockNumber) - if body == nil || len(body.Transactions) <= int(txIndex) { - log.Error("Transaction referenced missing", "number", blockNumber, "hash", blockHash, "index", txIndex) + blockNumber := ReadTxLookupEntry(db, hash) + if blockNumber == nil { + // return nil, common.Hash{}, 0, 0 + // TODO(daniel): delete the following old codes + // Old transaction representation, load the transaction and it's metadata separately + data, _ := db.Get(hash.Bytes()) + if len(data) == 0 { return nil, common.Hash{}, 0, 0 } - return body.Transactions[txIndex], blockHash, blockNumber, txIndex - } - // Old transaction representation, load the transaction and it's metadata separately - data, _ := db.Get(hash.Bytes()) - if len(data) == 0 { - return nil, common.Hash{}, 0, 0 + var tx types.Transaction + if err := rlp.DecodeBytes(data, &tx); err != nil { + return nil, common.Hash{}, 0, 0 + } + // Retrieve the blockchain positional metadata + data, _ = db.Get(append(hash.Bytes(), oldTxMetaSuffix...)) + if len(data) == 0 { + return nil, common.Hash{}, 0, 0 + } + var entry LegacyTxLookupEntry + if err := rlp.DecodeBytes(data, &entry); err != nil { + return nil, common.Hash{}, 0, 0 + } + return &tx, entry.BlockHash, entry.BlockIndex, entry.Index } - var tx types.Transaction - if err := rlp.DecodeBytes(data, &tx); err != nil { + blockHash := ReadCanonicalHash(db, *blockNumber) + if blockHash == (common.Hash{}) { return nil, common.Hash{}, 0, 0 } - // Retrieve the blockchain positional metadata - data, _ = db.Get(append(hash.Bytes(), oldTxMetaSuffix...)) - if len(data) == 0 { + body := ReadBody(db, blockHash, *blockNumber) + if body == nil { + log.Error("Transaction referenced missing", "number", blockNumber, "hash", blockHash) return nil, common.Hash{}, 0, 0 } - var entry TxLookupEntry - if err := rlp.DecodeBytes(data, &entry); err != nil { - return nil, common.Hash{}, 0, 0 + for txIndex, tx := range body.Transactions { + if tx.Hash() == hash { + return tx, blockHash, *blockNumber, uint64(txIndex) + } } - return &tx, entry.BlockHash, entry.BlockIndex, entry.Index + log.Error("Transaction not found", "number", blockNumber, "hash", blockHash, "txhash", hash) + return nil, common.Hash{}, 0, 0 } // ReadReceipt retrieves a specific transaction receipt from the database, along with // its added positional metadata. func ReadReceipt(db ethdb.Reader, hash common.Hash, config *params.ChainConfig) (*types.Receipt, common.Hash, uint64, uint64) { - // Retrieve the lookup metadata and resolve the receipt from the receipts - blockHash, blockNumber, receiptIndex := ReadTxLookupEntry(db, hash) - - if blockHash != (common.Hash{}) { - receipts := ReadReceipts(db, blockHash, blockNumber, config) - if len(receipts) <= int(receiptIndex) { - log.Error("Receipt refereced missing", "number", blockNumber, "hash", blockHash, "index", receiptIndex) + // Retrieve the context of the receipt based on the transaction hash + blockNumber := ReadTxLookupEntry(db, hash) + if blockNumber == nil { + // return nil, common.Hash{}, 0, 0 + // TODO(daniel): delete the following old codes + // Old receipt representation, load the receipt and set an unknown metadata + data, _ := db.Get(append(oldReceiptsPrefix, hash[:]...)) + if len(data) == 0 { return nil, common.Hash{}, 0, 0 } - return receipts[receiptIndex], blockHash, blockNumber, receiptIndex + var receipt types.ReceiptForStorage + err := rlp.DecodeBytes(data, &receipt) + if err != nil { + log.Error("Invalid receipt RLP", "hash", hash, "err", err) + return nil, common.Hash{}, 0, 0 + } + return (*types.Receipt)(&receipt), common.Hash{}, 0, 0 } - // Old receipt representation, load the receipt and set an unknown metadata - data, _ := db.Get(append(oldReceiptsPrefix, hash[:]...)) - if len(data) == 0 { + blockHash := ReadCanonicalHash(db, *blockNumber) + if blockHash == (common.Hash{}) { return nil, common.Hash{}, 0, 0 } - var receipt types.ReceiptForStorage - err := rlp.DecodeBytes(data, &receipt) - if err != nil { - log.Error("Invalid receipt RLP", "hash", hash, "err", err) + // Read all the receipts from the block and return the one with the matching hash + receipts := ReadReceipts(db, blockHash, *blockNumber, config) + for receiptIndex, receipt := range receipts { + if receipt.TxHash == hash { + return receipt, blockHash, *blockNumber, uint64(receiptIndex) + } } - return (*types.Receipt)(&receipt), common.Hash{}, 0, 0 + log.Error("Receipt not found", "number", blockNumber, "hash", blockHash, "txhash", hash) + return nil, common.Hash{}, 0, 0 } // ReadBloomBits retrieves the compressed bloom bit vector belonging to the given diff --git a/core/rawdb/accessors_indexes_test.go b/core/rawdb/accessors_indexes_test.go index 2412515d75ee..7f0dcf14185c 100644 --- a/core/rawdb/accessors_indexes_test.go +++ b/core/rawdb/accessors_indexes_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 The go-ethereum Authors +// Copyright 2018 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -22,46 +22,87 @@ import ( "github.com/XinFinOrg/XDPoSChain/common" "github.com/XinFinOrg/XDPoSChain/core/types" + "github.com/XinFinOrg/XDPoSChain/ethdb" + "github.com/XinFinOrg/XDPoSChain/rlp" ) // Tests that positional lookup metadata can be stored and retrieved. func TestLookupStorage(t *testing.T) { - db := NewMemoryDatabase() + tests := []struct { + name string + writeTxLookupEntries func(ethdb.Writer, *types.Block) + }{ + { + "DatabaseV6", + func(db ethdb.Writer, block *types.Block) { + WriteTxLookupEntriesByBlock(db, block) + }, + }, + { + "DatabaseV4-V5", + func(db ethdb.Writer, block *types.Block) { + for _, tx := range block.Transactions() { + db.Put(txLookupKey(tx.Hash()), block.Hash().Bytes()) + } + }, + }, + { + "DatabaseV3", + func(db ethdb.Writer, block *types.Block) { + for index, tx := range block.Transactions() { + entry := LegacyTxLookupEntry{ + BlockHash: block.Hash(), + BlockIndex: block.NumberU64(), + Index: uint64(index), + } + data, _ := rlp.EncodeToBytes(entry) + db.Put(txLookupKey(tx.Hash()), data) + } + }, + }, + } - tx1 := types.NewTransaction(1, common.BytesToAddress([]byte{0x11}), big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11, 0x11, 0x11}) - tx2 := types.NewTransaction(2, common.BytesToAddress([]byte{0x22}), big.NewInt(222), 2222, big.NewInt(22222), []byte{0x22, 0x22, 0x22}) - tx3 := types.NewTransaction(3, common.BytesToAddress([]byte{0x33}), big.NewInt(333), 3333, big.NewInt(33333), []byte{0x33, 0x33, 0x33}) - txs := []*types.Transaction{tx1, tx2, tx3} + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + db := NewMemoryDatabase() - block := types.NewBlock(&types.Header{Number: big.NewInt(314)}, txs, nil, nil) + tx1 := types.NewTransaction(1, common.BytesToAddress([]byte{0x11}), big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11, 0x11, 0x11}) + tx2 := types.NewTransaction(2, common.BytesToAddress([]byte{0x22}), big.NewInt(222), 2222, big.NewInt(22222), []byte{0x22, 0x22, 0x22}) + tx3 := types.NewTransaction(3, common.BytesToAddress([]byte{0x33}), big.NewInt(333), 3333, big.NewInt(33333), []byte{0x33, 0x33, 0x33}) + txs := []*types.Transaction{tx1, tx2, tx3} - // Check that no transactions entries are in a pristine database - for i, tx := range txs { - if txn, _, _, _ := ReadTransaction(db, tx.Hash()); txn != nil { - t.Fatalf("tx #%d [%x]: non existent transaction returned: %v", i, tx.Hash(), txn) - } - } - // Insert all the transactions into the database, and verify contents - WriteBlock(db, block) - WriteTxLookupEntriesByBlock(db, block) + block := types.NewBlock(&types.Header{Number: big.NewInt(314)}, txs, nil, nil) - for i, tx := range txs { - if txn, hash, number, index := ReadTransaction(db, tx.Hash()); txn == nil { - t.Fatalf("tx #%d [%x]: transaction not found", i, tx.Hash()) - } else { - if hash != block.Hash() || number != block.NumberU64() || index != uint64(i) { - t.Fatalf("tx #%d [%x]: positional metadata mismatch: have %x/%d/%d, want %x/%v/%v", i, tx.Hash(), hash, number, index, block.Hash(), block.NumberU64(), i) + // Check that no transactions entries are in a pristine database + for i, tx := range txs { + if txn, _, _, _ := ReadTransaction(db, tx.Hash()); txn != nil { + t.Fatalf("tx #%d [%x]: non existent transaction returned: %v", i, tx.Hash(), txn) + } } - if tx.String() != txn.String() { - t.Fatalf("tx #%d [%x]: transaction mismatch: have %v, want %v", i, tx.Hash(), txn, tx) + // Insert all the transactions into the database, and verify contents + WriteCanonicalHash(db, block.Hash(), block.NumberU64()) + WriteBlock(db, block) + tc.writeTxLookupEntries(db, block) + + for i, tx := range txs { + if txn, hash, number, index := ReadTransaction(db, tx.Hash()); txn == nil { + t.Fatalf("tx #%d [%x]: transaction not found", i, tx.Hash()) + } else { + if hash != block.Hash() || number != block.NumberU64() || index != uint64(i) { + t.Fatalf("tx #%d [%x]: positional metadata mismatch: have %x/%d/%d, want %x/%v/%v", i, tx.Hash(), hash, number, index, block.Hash(), block.NumberU64(), i) + } + if tx.Hash() != txn.Hash() { + t.Fatalf("tx #%d [%x]: transaction mismatch: have %v, want %v", i, tx.Hash(), txn, tx) + } + } } - } - } - // Delete the transactions and check purge - for i, tx := range txs { - DeleteTxLookupEntry(db, tx.Hash()) - if txn, _, _, _ := ReadTransaction(db, tx.Hash()); txn != nil { - t.Fatalf("tx #%d [%x]: deleted transaction returned: %v", i, tx.Hash(), txn) - } + // Delete the transactions and check purge + for i, tx := range txs { + DeleteTxLookupEntry(db, tx.Hash()) + if txn, _, _, _ := ReadTransaction(db, tx.Hash()); txn != nil { + t.Fatalf("tx #%d [%x]: deleted transaction returned: %v", i, tx.Hash(), txn) + } + } + }) } } diff --git a/core/rawdb/accessors_metadata.go b/core/rawdb/accessors_metadata.go index 857b4c2a3e49..38a62a5c8971 100644 --- a/core/rawdb/accessors_metadata.go +++ b/core/rawdb/accessors_metadata.go @@ -27,18 +27,30 @@ import ( "github.com/XinFinOrg/XDPoSChain/rlp" ) -// ReadDatabaseVersion reads the version number from db. -func ReadDatabaseVersion(db ethdb.KeyValueReader) int { - var vsn uint +// ReadDatabaseVersion retrieves the version number of the database. +func ReadDatabaseVersion(db ethdb.KeyValueReader) *uint64 { + var version uint64 + enc, _ := db.Get(databaseVersionKey) - rlp.DecodeBytes(enc, &vsn) - return int(vsn) + if len(enc) == 0 { + return nil + } + if err := rlp.DecodeBytes(enc, &version); err != nil { + return nil + } + + return &version } -// WriteDatabaseVersion writes vsn as the version number to db. -func WriteDatabaseVersion(db ethdb.KeyValueWriter, vsn int) { - enc, _ := rlp.EncodeToBytes(uint(vsn)) - db.Put(databaseVersionKey, enc) +// WriteDatabaseVersion stores the version number of the database +func WriteDatabaseVersion(db ethdb.KeyValueWriter, version uint64) { + enc, err := rlp.EncodeToBytes(version) + if err != nil { + log.Crit("Failed to encode database version", "err", err) + } + if err = db.Put(databaseVersionKey, enc); err != nil { + log.Crit("Failed to store the database version", "err", err) + } } // ReadChainConfig will fetch the network settings based on the given hash. diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 4136c495d05c..bd21557f5211 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -82,6 +82,14 @@ const ( freezerReceiptTable = "receipts" ) +// LegacyTxLookupEntry is the legacy TxLookupEntry definition with some unnecessary +// fields. +type LegacyTxLookupEntry struct { + BlockHash common.Hash + BlockIndex uint64 + Index uint64 +} + // encodeBlockNumber encodes a block number as big endian uint64 func encodeBlockNumber(number uint64) []byte { enc := make([]byte, 8) diff --git a/core/types/log.go b/core/types/log.go index bdee27e99bd0..5a2f388c4700 100644 --- a/core/types/log.go +++ b/core/types/log.go @@ -69,7 +69,8 @@ type rlpLog struct { Data []byte } -type rlpStorageLog struct { +// legacyRlpStorageLog is the previous storage encoding of a log including some redundant fields. +type legacyRlpStorageLog struct { Address common.Address Topics []common.Hash Data []byte @@ -82,7 +83,8 @@ type rlpStorageLog struct { // EncodeRLP implements rlp.Encoder. func (l *Log) EncodeRLP(w io.Writer) error { - return rlp.Encode(w, rlpLog{Address: l.Address, Topics: l.Topics, Data: l.Data}) + rl := rlpLog{Address: l.Address, Topics: l.Topics, Data: l.Data} + return rlp.Encode(w, &rl) } // DecodeRLP implements rlp.Decoder. @@ -105,32 +107,36 @@ type LogForStorage Log // EncodeRLP implements rlp.Encoder. func (l *LogForStorage) EncodeRLP(w io.Writer) error { - return rlp.Encode(w, rlpStorageLog{ - Address: l.Address, - Topics: l.Topics, - Data: l.Data, - BlockNumber: l.BlockNumber, - TxHash: l.TxHash, - TxIndex: l.TxIndex, - BlockHash: l.BlockHash, - Index: l.Index, - }) + rl := rlpLog{Address: l.Address, Topics: l.Topics, Data: l.Data} + return rlp.Encode(w, &rl) } // DecodeRLP implements rlp.Decoder. +// +// Note some redundant fields(e.g. block number, tx hash etc) will be assembled later. func (l *LogForStorage) DecodeRLP(s *rlp.Stream) error { - var dec rlpStorageLog - err := s.Decode(&dec) + blob, err := s.Raw() + if err != nil { + return err + } + var dec rlpLog + err = rlp.DecodeBytes(blob, &dec) if err == nil { *l = LogForStorage{ - Address: dec.Address, - Topics: dec.Topics, - Data: dec.Data, - BlockNumber: dec.BlockNumber, - TxHash: dec.TxHash, - TxIndex: dec.TxIndex, - BlockHash: dec.BlockHash, - Index: dec.Index, + Address: dec.Address, + Topics: dec.Topics, + Data: dec.Data, + } + } else { + // Try to decode log with previous definition. + var dec legacyRlpStorageLog + err = rlp.DecodeBytes(blob, &dec) + if err == nil { + *l = LogForStorage{ + Address: dec.Address, + Topics: dec.Topics, + Data: dec.Data, + } } } return err diff --git a/core/types/receipt.go b/core/types/receipt.go index d81730223cd7..54b107eaa07b 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -90,8 +90,25 @@ type receiptRLP struct { Logs []*Log } -// receiptStorageRLP is the original storage encoding of a receipt including some unnecessary fields. -type receiptStorageRLP struct { +// storedReceiptRLP is the storage encoding of a receipt. +type storedReceiptRLP struct { + PostStateOrStatus []byte + CumulativeGasUsed uint64 + Logs []*LogForStorage +} + +// v4StoredReceiptRLP is the storage encoding of a receipt used in database version 4. +type v4StoredReceiptRLP struct { + PostStateOrStatus []byte + CumulativeGasUsed uint64 + TxHash common.Hash + ContractAddress common.Address + Logs []*LogForStorage + GasUsed uint64 +} + +// v3StoredReceiptRLP is the previous storage encoding of a receipt including some unnecessary fields. +type v3StoredReceiptRLP struct { PostStateOrStatus []byte CumulativeGasUsed uint64 Bloom Bloom @@ -266,14 +283,10 @@ type ReceiptForStorage Receipt // EncodeRLP implements rlp.Encoder, and flattens all content fields of a receipt // into an RLP stream. func (r *ReceiptForStorage) EncodeRLP(w io.Writer) error { - enc := &receiptStorageRLP{ + enc := &storedReceiptRLP{ PostStateOrStatus: (*Receipt)(r).statusEncoding(), CumulativeGasUsed: r.CumulativeGasUsed, - Bloom: r.Bloom, - TxHash: r.TxHash, - ContractAddress: r.ContractAddress, Logs: make([]*LogForStorage, len(r.Logs)), - GasUsed: r.GasUsed, } for i, log := range r.Logs { enc.Logs[i] = (*LogForStorage)(log) @@ -289,28 +302,74 @@ func (r *ReceiptForStorage) DecodeRLP(s *rlp.Stream) error { if err != nil { return err } - return decodeStoredReceiptRLP(r, blob) + // Try decoding from the newest format for future proofness, then the older one + // for old nodes that just upgraded. V4 was an intermediate unreleased format so + // we do need to decode it, but it's not common (try last). + if err := decodeStoredReceiptRLP(r, blob); err == nil { + return nil + } + if err := decodeV3StoredReceiptRLP(r, blob); err == nil { + return nil + } + return decodeV4StoredReceiptRLP(r, blob) } func decodeStoredReceiptRLP(r *ReceiptForStorage, blob []byte) error { - var stored receiptStorageRLP + var stored storedReceiptRLP if err := rlp.DecodeBytes(blob, &stored); err != nil { return err } if err := (*Receipt)(r).setStatus(stored.PostStateOrStatus); err != nil { return err } - // Assign the consensus fields r.CumulativeGasUsed = stored.CumulativeGasUsed - r.Bloom = stored.Bloom r.Logs = make([]*Log, len(stored.Logs)) for i, log := range stored.Logs { r.Logs[i] = (*Log)(log) } - // Assign the implementation fields + r.Bloom = CreateBloom(Receipts{(*Receipt)(r)}) + + return nil +} + +func decodeV4StoredReceiptRLP(r *ReceiptForStorage, blob []byte) error { + var stored v4StoredReceiptRLP + if err := rlp.DecodeBytes(blob, &stored); err != nil { + return err + } + if err := (*Receipt)(r).setStatus(stored.PostStateOrStatus); err != nil { + return err + } + r.CumulativeGasUsed = stored.CumulativeGasUsed + r.TxHash = stored.TxHash + r.ContractAddress = stored.ContractAddress + r.GasUsed = stored.GasUsed + r.Logs = make([]*Log, len(stored.Logs)) + for i, log := range stored.Logs { + r.Logs[i] = (*Log)(log) + } + r.Bloom = CreateBloom(Receipts{(*Receipt)(r)}) + + return nil +} + +func decodeV3StoredReceiptRLP(r *ReceiptForStorage, blob []byte) error { + var stored v3StoredReceiptRLP + if err := rlp.DecodeBytes(blob, &stored); err != nil { + return err + } + if err := (*Receipt)(r).setStatus(stored.PostStateOrStatus); err != nil { + return err + } + r.CumulativeGasUsed = stored.CumulativeGasUsed + r.Bloom = stored.Bloom r.TxHash = stored.TxHash r.ContractAddress = stored.ContractAddress r.GasUsed = stored.GasUsed + r.Logs = make([]*Log, len(stored.Logs)) + for i, log := range stored.Logs { + r.Logs[i] = (*Log)(log) + } return nil } diff --git a/core/types/receipt_test.go b/core/types/receipt_test.go index 2fee4fa88fcb..0936e7340311 100644 --- a/core/types/receipt_test.go +++ b/core/types/receipt_test.go @@ -290,6 +290,14 @@ func TestLegacyReceiptDecoding(t *testing.T) { name string encode func(*Receipt) ([]byte, error) }{ + { + "StoredReceiptRLP", + encodeAsStoredReceiptRLP, + }, + { + "V4StoredReceiptRLP", + encodeAsV4StoredReceiptRLP, + }, { "V3StoredReceiptRLP", encodeAsV3StoredReceiptRLP, @@ -356,8 +364,35 @@ func TestLegacyReceiptDecoding(t *testing.T) { } } +func encodeAsStoredReceiptRLP(want *Receipt) ([]byte, error) { + stored := &storedReceiptRLP{ + PostStateOrStatus: want.statusEncoding(), + CumulativeGasUsed: want.CumulativeGasUsed, + Logs: make([]*LogForStorage, len(want.Logs)), + } + for i, log := range want.Logs { + stored.Logs[i] = (*LogForStorage)(log) + } + return rlp.EncodeToBytes(stored) +} + +func encodeAsV4StoredReceiptRLP(want *Receipt) ([]byte, error) { + stored := &v4StoredReceiptRLP{ + PostStateOrStatus: want.statusEncoding(), + CumulativeGasUsed: want.CumulativeGasUsed, + TxHash: want.TxHash, + ContractAddress: want.ContractAddress, + Logs: make([]*LogForStorage, len(want.Logs)), + GasUsed: want.GasUsed, + } + for i, log := range want.Logs { + stored.Logs[i] = (*LogForStorage)(log) + } + return rlp.EncodeToBytes(stored) +} + func encodeAsV3StoredReceiptRLP(want *Receipt) ([]byte, error) { - stored := &receiptStorageRLP{ + stored := &v3StoredReceiptRLP{ PostStateOrStatus: want.statusEncoding(), CumulativeGasUsed: want.CumulativeGasUsed, Bloom: want.Bloom, diff --git a/eth/backend.go b/eth/backend.go index 67afb6c6499d..dcfbf8c13731 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -151,15 +151,25 @@ func New(ctx *node.ServiceContext, config *ethconfig.Config, XDCXServ *XDCx.XDCX if lendingServ != nil { eth.Lending = lendingServ } - log.Info("Initialising Ethereum protocol", "versions", ProtocolVersions, "network", config.NetworkId) + + bcVersion := rawdb.ReadDatabaseVersion(chainDb) + var dbVer = "" + if bcVersion != nil { + dbVer = fmt.Sprintf("%d", *bcVersion) + } + log.Info("Initialising Ethereum protocol", "versions", ProtocolVersions, "network", config.NetworkId, "dbversion", dbVer) if !config.SkipBcVersionCheck { - bcVersion := rawdb.ReadDatabaseVersion(chainDb) - if bcVersion != core.BlockChainVersion && bcVersion != 0 { - return nil, fmt.Errorf("blockchain DB version mismatch (%d / %d). Run geth upgradedb", bcVersion, core.BlockChainVersion) + if bcVersion != nil && *bcVersion > core.BlockChainVersion { + return nil, fmt.Errorf("database version is v%d, not supports v%d", *bcVersion, core.BlockChainVersion) + } else if bcVersion == nil || *bcVersion < core.BlockChainVersion { + if bcVersion != nil { // only print warning on upgrade, not on init + log.Warn("Upgrade blockchain database version", "from", dbVer, "to", core.BlockChainVersion) + } + rawdb.WriteDatabaseVersion(chainDb, core.BlockChainVersion) } - rawdb.WriteDatabaseVersion(chainDb, core.BlockChainVersion) } + var ( vmConfig = vm.Config{EnablePreimageRecording: config.EnablePreimageRecording} cacheConfig = &core.CacheConfig{Disabled: config.NoPruning, TrieNodeLimit: config.TrieCache, TrieTimeLimit: config.TrieTimeout} diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index a578a34f7178..e1a87796bb78 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -23,7 +23,6 @@ import ( "testing" "github.com/XinFinOrg/XDPoSChain/core/rawdb" - "github.com/XinFinOrg/XDPoSChain/common" "github.com/XinFinOrg/XDPoSChain/consensus/ethash" "github.com/XinFinOrg/XDPoSChain/core" diff --git a/les/handler.go b/les/handler.go index 34f5ad2da944..bef0fd64678f 100644 --- a/les/handler.go +++ b/les/handler.go @@ -1172,9 +1172,9 @@ func (pm *ProtocolManager) txStatus(hashes []common.Hash) []txStatus { // If the transaction is unknown to the pool, try looking it up locally if stat == txpool.TxStatusUnknown { - if block, number, index := rawdb.ReadTxLookupEntry(pm.chainDb, hashes[i]); block != (common.Hash{}) { + if tx, blockHash, blockNumber, txIndex := rawdb.ReadTransaction(pm.chainDb, hashes[i]); tx != nil { stats[i].Status = txpool.TxStatusIncluded - stats[i].Lookup = &rawdb.TxLookupEntry{BlockHash: block, BlockIndex: number, Index: index} + stats[i].Lookup = &rawdb.LegacyTxLookupEntry{BlockHash: blockHash, BlockIndex: blockNumber, Index: txIndex} } } } diff --git a/les/handler_test.go b/les/handler_test.go index 609024f5a172..d6b6927b36ac 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -557,8 +557,8 @@ func TestTransactionStatusLes2(t *testing.T) { // check if their status is included now block1hash := rawdb.ReadCanonicalHash(db, 1) - test(tx1, false, txStatus{Status: txpool.TxStatusIncluded, Lookup: &rawdb.TxLookupEntry{BlockHash: block1hash, BlockIndex: 1, Index: 0}}) - test(tx2, false, txStatus{Status: txpool.TxStatusIncluded, Lookup: &rawdb.TxLookupEntry{BlockHash: block1hash, BlockIndex: 1, Index: 1}}) + test(tx1, false, txStatus{Status: txpool.TxStatusIncluded, Lookup: &rawdb.LegacyTxLookupEntry{BlockHash: block1hash, BlockIndex: 1, Index: 0}}) + test(tx2, false, txStatus{Status: txpool.TxStatusIncluded, Lookup: &rawdb.LegacyTxLookupEntry{BlockHash: block1hash, BlockIndex: 1, Index: 1}}) // create a reorg that rolls them back gchain, _ = core.GenerateChain(params.TestChainConfig, chain.GetBlockByNumber(0), ethash.NewFaker(), db, 2, func(i int, block *core.BlockGen) {}) diff --git a/les/odr_test.go b/les/odr_test.go index 74e944098210..d47f0ee71f4d 100644 --- a/les/odr_test.go +++ b/les/odr_test.go @@ -197,6 +197,9 @@ func testOdr(t *testing.T, protocol int, expFail uint64, fn odrTestFn) { lpm.synchronise(lpeer) test := func(expFail uint64) { + // Mark this as a helper to put the failures at the correct lines + t.Helper() + for i := uint64(0); i <= pm.blockchain.CurrentHeader().Number.Uint64(); i++ { bhash := rawdb.ReadCanonicalHash(db, i) b1 := fn(light.NoOdr, db, pm.chainConfig, pm.blockchain.(*core.BlockChain), nil, bhash) @@ -208,10 +211,10 @@ func testOdr(t *testing.T, protocol int, expFail uint64, fn odrTestFn) { eq := bytes.Equal(b1, b2) exp := i < expFail if exp && !eq { - t.Errorf("odr mismatch") + t.Fatalf("odr mismatch: have %x, want %x", b2, b1) } if !exp && eq { - t.Errorf("unexpected odr match") + t.Fatalf("unexpected odr match") } } } @@ -221,6 +224,7 @@ func testOdr(t *testing.T, protocol int, expFail uint64, fn odrTestFn) { peers.Unregister(lpeer.id) time.Sleep(time.Millisecond * 10) // ensure that all peerSetNotify callbacks are executed test(expFail) + // expect all retrievals to pass peers.Register(lpeer) time.Sleep(time.Millisecond * 10) // ensure that all peerSetNotify callbacks are executed @@ -228,6 +232,7 @@ func testOdr(t *testing.T, protocol int, expFail uint64, fn odrTestFn) { lpeer.hasBlock = func(common.Hash, uint64) bool { return true } lpeer.lock.Unlock() test(5) + // still expect all retrievals to pass, now data should be cached locally peers.Unregister(lpeer.id) time.Sleep(time.Millisecond * 10) // ensure that all peerSetNotify callbacks are executed diff --git a/les/protocol.go b/les/protocol.go index 42c984204185..68637cff8d97 100644 --- a/les/protocol.go +++ b/les/protocol.go @@ -225,6 +225,6 @@ type proofsData [][]rlp.RawValue type txStatus struct { Status txpool.TxStatus - Lookup *rawdb.TxLookupEntry `rlp:"nil"` + Lookup *rawdb.LegacyTxLookupEntry `rlp:"nil"` Error string } diff --git a/light/odr_util.go b/light/odr_util.go index 60e538448b0b..4dd44f8433fe 100644 --- a/light/odr_util.go +++ b/light/odr_util.go @@ -170,13 +170,10 @@ func GetBlockReceipts(ctx context.Context, odr OdrBackend, hash common.Hash, num // GetBlockLogs retrieves the logs generated by the transactions included in a // block given by its hash. Logs will be filled in with context data. func GetBlockLogs(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) ([][]*types.Log, error) { - receipts, _ := GetBlockReceipts(ctx, odr, hash, number) - if receipts == nil { - r := &ReceiptsRequest{Hash: hash, Number: number} - if err := odr.Retrieve(ctx, r); err != nil { - return nil, err - } - receipts = r.Receipts + // Retrieve the potentially incomplete receipts from disk or network + receipts, err := GetBlockReceipts(ctx, odr, hash, number) + if err != nil { + return nil, err } // Return the logs without deriving any computed fields on the receipts logs := make([][]*types.Log, len(receipts))