From a262188ca09d5189b684f5939a450e31477a9fb5 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sat, 7 May 2022 04:54:20 -0500 Subject: [PATCH] blockchain: Implement header proof storage. This modifies the chain logic to create and store the individual commitment hashes covered by the commitment root field of the header of each block and also adds code to migrate the database to retroactively create and store entries for all applicable historical blocks. The upgrade can be interrupted at any point and future invocations will resume from the point it was interrupted. The following is a high level overview of the changes: - Introduce a new database bucket to house the header commitments - Add serialization code for use when storing and loading the individual header commitment hashes - Add full test coverage for new serialization code - Store the commitment hashes in the db when connecting blocks - Implement database migration code to retroactively store the commitment hashes for all applicable historical blocks - Bump the chain database version to 13 - Support resuming from interrupted upgrades - Add a new func on the internal header commitment data struct that returns the v1 header commitment hashes to consolidate the logic - Update FilterByBlockHash to load the header commitments from the db and generate the inclusion proof accordingly --- blockchain/chain.go | 31 ++- blockchain/chainio.go | 125 ++++++++++- blockchain/chainio_test.go | 95 ++++++++ blockchain/go.mod | 2 +- blockchain/headercmt.go | 36 ++- blockchain/upgrade.go | 441 +++++++++++++++++++++++++++++++++++++ 6 files changed, 708 insertions(+), 22 deletions(-) diff --git a/blockchain/chain.go b/blockchain/chain.go index 3d13088270..d5ffe8fb2c 100644 --- a/blockchain/chain.go +++ b/blockchain/chain.go @@ -677,14 +677,18 @@ func (b *BlockChain) connectBlock(node *blockNode, block, parent *dcrutil.Block, return err } - // NOTE: When more header commitments are added, the inclusion proofs - // will need to be generated and stored to the database here (when not - // already stored). There is no need to store them currently because - // there is only a single commitment which means there are no sibling - // hashes that typically form the inclusion proofs due to the fact a - // single leaf merkle tree reduces to having the same root as the leaf - // and therefore the proof only consists of checking the leaf hash - // itself against the commitment root. + // Determine the individual commitment hashes that comprise the leaves of + // the header commitment merkle tree depending on the active agendas. These + // are stored in the database below so that inclusion proofs can be + // generated for each commitment. + var hdrCommitmentLeaves []chainhash.Hash + hdrCommitmentsActive, err := b.isHeaderCommitmentsAgendaActive(node.parent) + if err != nil { + return err + } + if hdrCommitmentsActive { + hdrCommitmentLeaves = hdrCommitments.v1Leaves() + } // Generate a new best state snapshot that will be used to update the // database and later memory if all database updates are successful. @@ -741,6 +745,13 @@ func (b *BlockChain) connectBlock(node *blockNode, block, parent *dcrutil.Block, return err } + // Store the leaf hashes of the header commitment merkle tree in the + // database. Nothing is written when there aren't any. + err = dbPutHeaderCommitments(dbTx, block.Hash(), hdrCommitmentLeaves) + if err != nil { + return err + } + return nil }) if err != nil { @@ -923,6 +934,10 @@ func (b *BlockChain) disconnectBlock(node *blockNode, block, parent *dcrutil.Blo // NOTE: The GCS filter is intentionally not removed on disconnect to // ensure that lightweight clients still have access to them if they // happen to be on a side chain after coming back online after a reorg. + // + // Similarly, the commitment hashes needed to generate the associated + // inclusion proof for the header commitment are not removed for the + // same reason. return nil }) diff --git a/blockchain/chainio.go b/blockchain/chainio.go index fbd08f4fc1..5d964f394b 100644 --- a/blockchain/chainio.go +++ b/blockchain/chainio.go @@ -26,7 +26,7 @@ import ( const ( // currentDatabaseVersion indicates the current database version. - currentDatabaseVersion = 12 + currentDatabaseVersion = 13 // currentBlockIndexVersion indicates the current block index database // version. @@ -96,6 +96,11 @@ var ( // filters. gcsFilterBucketName = []byte("gcsfilters") + // headerCmtsBucketName is the name of the db bucket used to house header + // commitment journal entries which consist of the hashes that the + // commitment root field of blocks commit to. + headerCmtsBucketName = []byte("hdrcmts") + // treasuryBucketName is the name of the db bucket that is used to house // TADD/TSPEND additions and subtractions from the treasury account. treasuryBucketName = []byte("treasury") @@ -854,6 +859,118 @@ func dbPutGCSFilter(dbTx database.Tx, blockHash *chainhash.Hash, filter *gcs.Fil return filterBucket.Put(blockHash[:], serialized) } +// ----------------------------------------------------------------------------- +// The header commitments journal consists of an entry for each block connected +// to the main chain (or has ever been connected to it) that contains each of +// the individual commitments covered by the commitment root field of the header +// of that block. +// +// Note that there will also not be an entry for blocks that do not commit to +// anything such as those prior to the activation of the header commitments +// agenda on networks where it is not always active. +// +// The serialized key format is: +// +// +// +// Field Type Size +// block hash chainhash.Hash chainhash.HashSize +// +// The serialized value format is: +// +// +// +// Field Type Size +// num commitment hashes VLQ variable +// commitment hashes +// commitment hash chainhash.Hash chainhash.HashSize +// +// ----------------------------------------------------------------------------- + +// serializeHeaderCommitments serializes the passed commitment hashes into a +// single byte slice according to the format described in detail above. +func serializeHeaderCommitments(commitments []chainhash.Hash) []byte { + // Nothing to serialize when there are no commitments. + if len(commitments) == 0 { + return nil + } + + // Calculate the full size needed to serialize the commitments. + numCommitments := len(commitments) + serializedLen := serializeSizeVLQ(uint64(numCommitments)) + + numCommitments*chainhash.HashSize + + // Serialize the commitments. + serialized := make([]byte, serializedLen) + offset := putVLQ(serialized, uint64(numCommitments)) + for i := range commitments { + copy(serialized[offset:], commitments[i][:]) + offset += chainhash.HashSize + } + return serialized +} + +// deserializeHeaderCommitments decodes the passed serialized byte slice into a +// slice of commitment hashes according to the format described in detail above. +func deserializeHeaderCommitments(serialized []byte) ([]chainhash.Hash, error) { + // Nothing is serialized when there are no commitments. + if len(serialized) == 0 { + return nil, nil + } + + // Deserialize the number of commitments. + numCommitments, offset := deserializeVLQ(serialized) + if offset >= len(serialized) { + str := "unexpected end of data after num commitments" + return nil, makeDbErr(database.ErrCorruption, str) + } + + // Ensure there are enough bytes remaining to read for the expected number + // of commitments. + totalCommitmentsSize := int(numCommitments) * chainhash.HashSize + if len(serialized[offset:]) < totalCommitmentsSize { + str := fmt.Sprintf("unexpected end of data after number of commitments "+ + "(got %v, need %v)", len(serialized[offset:]), totalCommitmentsSize) + return nil, makeDbErr(database.ErrCorruption, str) + } + + // Deserialize the commitments. + commitments := make([]chainhash.Hash, numCommitments) + for i := 0; i < int(numCommitments); i++ { + copy(commitments[i][:], serialized[offset:offset+chainhash.HashSize]) + offset += chainhash.HashSize + } + + return commitments, nil +} + +// dbFetchHeaderCommitments fetches the hashes that the commitment root field of +// the header commits to for the passed block. +// +// When there is no entry for the provided block hash, nil will be returned for +// both the commitment hashes and the error. +func dbFetchHeaderCommitments(dbTx database.Tx, blockHash *chainhash.Hash) ([]chainhash.Hash, error) { + commitmentsBucket := dbTx.Metadata().Bucket(headerCmtsBucketName) + serialized := commitmentsBucket.Get(blockHash[:]) + return deserializeHeaderCommitments(serialized) +} + +// dbPutHeaderCommitments uses an existing database transaction to update the +// hashes that the commitment root field of the header commits to for the passed +// block. +// +// No database entry will be created when the provided commitments slice is nil +// or empty (aka zero length). +func dbPutHeaderCommitments(dbTx database.Tx, blockHash *chainhash.Hash, commitments []chainhash.Hash) error { + serialized := serializeHeaderCommitments(commitments) + if len(serialized) == 0 { + return nil + } + + commitmentsBucket := dbTx.Metadata().Bucket(headerCmtsBucketName) + return commitmentsBucket.Put(blockHash[:], serialized) +} + // ----------------------------------------------------------------------------- // The database information contains information about the version and date // of the blockchain database. @@ -1233,6 +1350,12 @@ func (b *BlockChain) createChainState() error { return err } _, err = meta.CreateBucket(treasuryTSpendBucketName) + if err != nil { + return err + } + + // Create the bucket that houses the header commitments. + _, err = meta.CreateBucket(headerCmtsBucketName) return err }) return err diff --git a/blockchain/chainio_test.go b/blockchain/chainio_test.go index adeaf26d14..103755e472 100644 --- a/blockchain/chainio_test.go +++ b/blockchain/chainio_test.go @@ -792,6 +792,101 @@ func TestSpendJournalErrors(t *testing.T) { } } +// TestHeaderCommitmentSerialization ensures serializing and deserializing +// header commitment journal entries works as expected. +func TestHeaderCommitmentSerialization(t *testing.T) { + t.Parallel() + + cmtOneHash := *mustParseHash("0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20") + cmtTwoHash := *mustParseHash("02030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f2021") + tests := []struct { + name string + commitments []chainhash.Hash + serialized []byte + }{{ + name: "no commitments", + commitments: nil, + serialized: nil, + }, { + name: "one commitment", + commitments: []chainhash.Hash{cmtOneHash}, + serialized: hexToBytes("01" + + "201f1e1d1c1b1a191817161514131211100f0e0d0c0b0a090807060504030201"), + }, { + name: "two commitmentments", + commitments: []chainhash.Hash{cmtOneHash, cmtTwoHash}, + serialized: hexToBytes("02" + + "201f1e1d1c1b1a191817161514131211100f0e0d0c0b0a090807060504030201" + + "21201f1e1d1c1b1a191817161514131211100f0e0d0c0b0a0908070605040302"), + }} + + for _, test := range tests { + // Ensure the commitments serialize to the expected value. + gotBytes := serializeHeaderCommitments(test.commitments) + if !bytes.Equal(gotBytes, test.serialized) { + t.Errorf("%q: mismatched bytes - got %x, want %x", test.name, + gotBytes, test.serialized) + continue + } + + // Ensure the serialized bytes are decoded back to the expected + // commitments. + commitments, err := deserializeHeaderCommitments(test.serialized) + if err != nil { + t.Errorf("%q: unexpected error: %v", test.name, err) + continue + } + if !reflect.DeepEqual(commitments, test.commitments) { + t.Errorf("%q: mismatched commitments - got %v, want %v", test.name, + commitments, test.commitments) + continue + } + } +} + +// TestHeaderCommitmentDeserializeErrors peforms negative tests against +// deserializing header commitment journal entries to ensure error paths work as +// expected. +func TestHeaderCommitmentDeserializeErrors(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + serialized []byte + err error + }{{ + name: "short data in number of commitments", + serialized: hexToBytes("80"), + err: database.ErrCorruption, + }, { + name: "short data in commitment hashes", + serialized: hexToBytes("01" + + "201f1e1d1c1b1a191817161514131211100f0e0d0c0b0a0908070605040302"), + err: database.ErrCorruption, + }, { + name: "short data in commitment hashes 2 begin", + serialized: hexToBytes("02" + + "201f1e1d1c1b1a191817161514131211100f0e0d0c0b0a090807060504030201"), + err: database.ErrCorruption, + }, { + name: "short data in commitment hashes 2 end", + serialized: hexToBytes("02" + + "201f1e1d1c1b1a191817161514131211100f0e0d0c0b0a090807060504030201" + + "21201f1e1d1c1b1a191817161514131211100f0e0d0c0b0a09080706050403"), + err: database.ErrCorruption, + }} + + for _, test := range tests { + // Ensure the expected error type and code is returned. + _, err := deserializeHeaderCommitments(test.serialized) + if !errors.Is(err, test.err) { + t.Errorf("%q: wrong error -- got: %v, want: %v", test.name, err, + test.err) + continue + } + } +} + // TestBestChainStateSerialization ensures serializing and deserializing the // best chain state works as expected. func TestBestChainStateSerialization(t *testing.T) { diff --git a/blockchain/go.mod b/blockchain/go.mod index ab3927220c..67e8f7ba91 100644 --- a/blockchain/go.mod +++ b/blockchain/go.mod @@ -7,6 +7,7 @@ require ( github.com/decred/dcrd/blockchain/standalone/v2 v2.1.0 github.com/decred/dcrd/chaincfg/chainhash v1.0.3 github.com/decred/dcrd/chaincfg/v3 v3.1.1 + github.com/decred/dcrd/crypto/blake256 v1.0.0 github.com/decred/dcrd/database/v3 v3.0.0 github.com/decred/dcrd/dcrec v1.0.0 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 @@ -23,7 +24,6 @@ require ( github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect github.com/dchest/siphash v1.2.2 // indirect github.com/decred/base58 v1.0.3 // indirect - github.com/decred/dcrd/crypto/blake256 v1.0.0 // indirect github.com/decred/dcrd/crypto/ripemd160 v1.0.1 // indirect github.com/decred/dcrd/dcrec/edwards/v2 v2.0.2 // indirect github.com/golang/snappy v0.0.4 // indirect diff --git a/blockchain/headercmt.go b/blockchain/headercmt.go index c02012d75e..62fd1987d3 100644 --- a/blockchain/headercmt.go +++ b/blockchain/headercmt.go @@ -7,6 +7,7 @@ package blockchain import ( "fmt" + "github.com/decred/dcrd/blockchain/standalone/v2" "github.com/decred/dcrd/chaincfg/chainhash" "github.com/decred/dcrd/database/v3" "github.com/decred/dcrd/dcrutil/v4" @@ -26,6 +27,12 @@ type headerCommitmentData struct { filterHash chainhash.Hash } +// v1Leaves returns the individual commitment hashes that comprise the leaves of +// the merkle tree for a v1 header commitment. +func (c *headerCommitmentData) v1Leaves() []chainhash.Hash { + return []chainhash.Hash{c.filterHash} +} + // CalcCommitmentRootV1 calculates and returns the required v1 block commitment // root from the filter hash it commits to. // @@ -153,29 +160,34 @@ func (b *BlockChain) FilterByBlockHash(hash *chainhash.Hash) (*gcs.FilterV2, *He return nil, nil, contextError(ErrNoFilter, str) } + // Attempt to load the filter and associated header commitments from the + // database. var filter *gcs.FilterV2 + var leaves []chainhash.Hash err := b.db.View(func(dbTx database.Tx) error { var err error filter, err = dbFetchGCSFilter(dbTx, hash) + if err != nil { + return err + } + if filter == nil { + str := fmt.Sprintf("no filter available for block %s", hash) + return contextError(ErrNoFilter, str) + } + + leaves, err = dbFetchHeaderCommitments(dbTx, hash) return err }) if err != nil { return nil, nil, err } - if filter == nil { - str := fmt.Sprintf("no filter available for block %s", hash) - return nil, nil, contextError(ErrNoFilter, str) - } - // NOTE: When more header commitments are added, this will need to load the - // inclusion proof for the filter from the database. However, since there - // is only currently a single commitment, there is only a single leaf in the - // commitment merkle tree, and hence the proof hashes will always be empty - // given there are no siblings. Adding an additional header commitment will - // require a consensus vote anyway and this can be updated at that time. + // Generate the header commitment inclusion proof for the filter. + const proofIndex = HeaderCmtFilterIndex + proof := standalone.GenerateInclusionProof(leaves, proofIndex) headerProof := &HeaderProof{ - ProofIndex: HeaderCmtFilterIndex, - ProofHashes: nil, + ProofIndex: proofIndex, + ProofHashes: proof, } return filter, headerProof, nil } diff --git a/blockchain/upgrade.go b/blockchain/upgrade.go index f394bcc795..ebec357fbb 100644 --- a/blockchain/upgrade.go +++ b/blockchain/upgrade.go @@ -18,6 +18,7 @@ import ( "github.com/decred/dcrd/blockchain/stake/v5" "github.com/decred/dcrd/chaincfg/chainhash" "github.com/decred/dcrd/chaincfg/v3" + "github.com/decred/dcrd/crypto/blake256" "github.com/decred/dcrd/database/v3" "github.com/decred/dcrd/dcrec/secp256k1/v4" "github.com/decred/dcrd/gcs/v4" @@ -4647,6 +4648,436 @@ func upgradeToVersion12(ctx context.Context, db database.DB, chainParams *chainc return nil } +// serializeHeaderCommitmentsV1 serializes the passed commitment hashes into a +// single byte slice according to the version 1 format. +func serializeHeaderCommitmentsV1(commitments []chainhash.Hash) []byte { + // Nothing to serialize when there are no commitments. + if len(commitments) == 0 { + return nil + } + + // Calculate the full size needed to serialize the commitments. + numCommitments := len(commitments) + serializedLen := serializeSizeVLQ(uint64(numCommitments)) + + numCommitments*chainhash.HashSize + + // Serialize the commitments. + serialized := make([]byte, serializedLen) + offset := putVLQ(serialized, uint64(numCommitments)) + for i := range commitments { + copy(serialized[offset:], commitments[i][:]) + offset += chainhash.HashSize + } + return serialized +} + +// initializeHeaderCmts creates and stores the hashes that comprise the header +// commitments for all applicable blocks. This ensures the commitments are +// immediately available for use when constructing inclusion proofs for clients +// and simplifies the rest of the related code since it can rely on the data +// being available once the upgrade completes. +// +// The database is guaranteed to have the header commitments for all blocks +// after the point that header commitments activated and also have both the +// block data data and associated filter data available. In practice, that +// includes all blocks that are part of the main chain as well as any side chain +// blocks that were fully connected at some point. +func initializeHeaderCmts(ctx context.Context, db database.DB, params *chaincfg.Params) error { + log.Info("Storing header commitments. This may take a while...") + start := time.Now() + + // Hardcoded data so updates do not affect old upgrades. + const hashSize = chainhash.HashSize + v1ChainStateKeyName := []byte("chainstate") + v3BlockIdxBucketName := []byte("blockidxv3") + gcsBucketName := []byte("gcsfilters") + hdrCmtProgressKeyName := []byte("hdrcmtprogress") + byteOrder := binary.LittleEndian + + // Determine the current best chain tip using the version 1 chain state. + var bestTipHash chainhash.Hash + var bestTipHeight uint32 + err := db.View(func(dbTx database.Tx) error { + // Load the current best chain tip hash and height from the v1 chain + // state. + // + // The serialized format of the v1 chain state is roughly: + // + // + // + // Field Type Size + // block hash chainhash.Hash hashSize + // block height uint32 4 bytes + // rest of data... + meta := dbTx.Metadata() + serializedChainState := meta.Get(v1ChainStateKeyName) + if serializedChainState == nil { + str := fmt.Sprintf("chain state with key %s does not exist", + v1ChainStateKeyName) + return errDeserialize(str) + } + if len(serializedChainState) < hashSize+4 { + str := "version 1 chain state is malformed" + return errDeserialize(str) + } + copy(bestTipHash[:], serializedChainState[0:hashSize]) + offset := hashSize + bestTipHeight = byteOrder.Uint32(serializedChainState[offset : offset+4]) + return nil + }) + if err != nil { + return err + } + + if interruptRequested(ctx) { + return errInterruptRequested + } + + // Determine the minimum block height that possibly needs to have header + // commitments stored since it only applies to blocks once the header + // commitments agenda activated. However, determining if the agenda is + // active and the point at which it activated requires a bunch of data that + // is not readily available here in the upgrade code, so hard code the data + // for the main and test networks to significantly optimize those cases and + // just check everything for other networks. While this does potentially do + // more work than is strictly necessary for those other networks, it is + // quite rare for them to have old databases in practice, so it is a + // reasonable tradeoff. + storeFromHeight := uint32(1) + switch params.Net { + case wire.MainNet: + storeFromHeight = 431488 + case wire.TestNet3: + storeFromHeight = 323328 + } + + // There is nothing to do if the best chain tip is prior to the point after + // where header commitments need to be stored. + if bestTipHeight < storeFromHeight { + return nil + } + + // Resume from in-progress upgrades. + err = db.View(func(dbTx database.Tx) error { + progressHeightBytes := dbTx.Metadata().Get(hdrCmtProgressKeyName) + if progressHeightBytes != nil { + storeFromHeight = byteOrder.Uint32(progressHeightBytes) + log.Infof("Resuming upgrade at height %d", storeFromHeight+1) + } + return nil + }) + if err != nil { + return err + } + + // blockTreeEntry represents a version 3 block index entry with the details + // needed to be able to determine which blocks need to have the header + // commitments stored as well as which ones comprise the main chain. + type blockTreeEntry struct { + parent *blockTreeEntry + children []*blockTreeEntry + hash chainhash.Hash + height uint32 + mainChain bool + cmtRoot chainhash.Hash + status byte + } + + // Load the block tree to determine all blocks that need to have the header + // commitments stored using the version 3 block index along with the + // information determined above. + var startParent *blockTreeEntry + blockTree := make(map[chainhash.Hash]*blockTreeEntry) + err = db.View(func(dbTx database.Tx) error { + // Hardcoded data so updates do not affect old upgrades. + const blockHdrSize = 180 + + // Construct a full block tree from the version 3 block index by mapping + // each block to its parent block. + var lastEntry, parent *blockTreeEntry + meta := dbTx.Metadata() + v3BlockIdxBucket := meta.Bucket(v3BlockIdxBucketName) + if v3BlockIdxBucket == nil { + return fmt.Errorf("bucket %s does not exist", v3BlockIdxBucketName) + } + cursor := v3BlockIdxBucket.Cursor() + for ok := cursor.First(); ok; ok = cursor.Next() { + if interruptRequested(ctx) { + return errInterruptRequested + } + + // Deserialize the header from the version 3 block index entry. + // + // The serialized value format of a v3 block index entry is roughly: + // + // + // + // Field Type Size + // block header wire.BlockHeader 180 bytes + // status byte 1 byte + // rest of data... + serializedBlkIdxEntry := cursor.Value() + if len(serializedBlkIdxEntry) < blockHdrSize { + return errDeserialize("unexpected end of data while reading " + + "block header") + } + hB := serializedBlkIdxEntry[0:blockHdrSize] + var header wire.BlockHeader + if err := header.Deserialize(bytes.NewReader(hB)); err != nil { + return err + } + if blockHdrSize+1 > len(serializedBlkIdxEntry) { + return errDeserialize("unexpected end of data while reading " + + "status") + } + status := serializedBlkIdxEntry[blockHdrSize] + + // Stop loading if the best chain tip height is exceeded. This + // can happen when there are block index entries for known headers + // that haven't had their associated block data fully validated and + // connected yet and therefore are not relevant for this upgrade + // code. + blockHeight := header.Height + if blockHeight > bestTipHeight { + break + } + + // Determine the parent block node. Since the entries are iterated + // in order of height, there is a very good chance the previous + // one processed is the parent. + blockHash := header.BlockHash() + if lastEntry == nil { + if blockHash != params.GenesisHash { + str := fmt.Sprintf("initializeHeaderCmts: expected first "+ + "entry to be genesis block, found %s", blockHash) + return errDeserialize(str) + } + } else if header.PrevBlock == lastEntry.hash { + parent = lastEntry + } else { + parent = blockTree[header.PrevBlock] + if parent == nil { + str := fmt.Sprintf("initializeHeaderCmts: could not find "+ + "parent for block %s", blockHash) + return errDeserialize(str) + } + } + + // Add the block to the block tree. + treeEntry := &blockTreeEntry{ + parent: parent, + hash: blockHash, + height: blockHeight, + cmtRoot: header.StakeRoot, + status: status, + } + blockTree[blockHash] = treeEntry + if parent != nil { + parent.children = append(parent.children, treeEntry) + } + + // Determine the parent of the block to start storing the header + // commitments. + if blockHeight == storeFromHeight && startParent == nil { + startParent = parent + } + + lastEntry = treeEntry + } + + // Determine the blocks that comprise the main chain by starting at the + // best tip and walking backwards to the oldest tracked block. + bestTip := blockTree[bestTipHash] + if bestTip == nil { + str := fmt.Sprintf("chain tip %s is not in block index", bestTipHash) + return errDeserialize(str) + } + for entry := bestTip; entry != nil; entry = entry.parent { + entry.mainChain = true + } + return nil + }) + if err != nil { + return err + } + + // Start from block 1 when no other starting block was found above. + if startParent == nil { + startParent = blockTree[params.GenesisHash] + } + + if interruptRequested(ctx) { + return errInterruptRequested + } + + // Create the new header commitments bucket as needed. + headerCmtsBucketName := []byte("hdrcmts") + err = db.Update(func(dbTx database.Tx) error { + _, err := dbTx.Metadata().CreateBucketIfNotExists(headerCmtsBucketName) + return err + }) + if err != nil { + return err + } + + // doBatch contains the primary logic for storing the header commitments + // when moving from database version 12 to 13 in batches. This is done + // because attempting to store them all in a single database transaction + // could result in massive memory usage and could potentially crash on many + // systems due to ulimits. + // + // It returns whether or not all entries have been stored. + const maxEntries = 20000 + processQueue := make([]*blockTreeEntry, 0, 5) + processQueue = append(processQueue, startParent.children...) + var totalStored uint64 + doBatch := func(dbTx database.Tx) (bool, error) { + meta := dbTx.Metadata() + commitmentsBucket := meta.Bucket(headerCmtsBucketName) + if commitmentsBucket == nil { + return false, fmt.Errorf("bucket %s does not exist", + headerCmtsBucketName) + } + filterBucket := meta.Bucket(gcsBucketName) + if filterBucket == nil { + return false, fmt.Errorf("bucket %s does not exist", gcsBucketName) + } + + var logProgress bool + var numStored uint64 + err := func() error { + for len(processQueue) > 0 { + if interruptRequested(ctx) { + logProgress = true + return errInterruptRequested + } + + if numStored >= maxEntries { + logProgress = true + return errBatchFinished + } + + node := processQueue[0] + processQueue = processQueue[1:] + processQueue = append(processQueue, node.children...) + + // Skip side chain blocks that do not have block data available + // since they do not have the filters stored. + const v3StatusDataStored = 1 << 0 + if node.status&v3StatusDataStored == 0 { + if node.mainChain { + return AssertError(fmt.Sprintf("no block data for "+ + "main chain block %s (height %d)", node.hash, + node.height)) + } + continue + } + + // Attempt to load the filter for the block while ensuring that + // the data is available for main chain blocks and skipping any + // side chain blocks that do not have it available. Blocks that + // were never part of the main chain will not have any filter + // data stored. + filterBytes := filterBucket.Get(node.hash[:]) + if len(filterBytes) == 0 { + if node.mainChain { + return AssertError(fmt.Sprintf("no filter is stored "+ + "for main chain block %s (height %d)", node.hash, + node.height)) + } + continue + } + + // Store the filter hash as the one and only commitment when the + // header commits to it. + // + // This approach is used because the headers for all blocks at + // database version 12 only committed to the filter once the + // header commitments agenda activated and agenda information is + // not readily available here in the upgrade code. + filterHash := blake256.Sum256(filterBytes) + if filterHash != node.cmtRoot { + continue + } + + // Store the commitment in the database. + commitments := []chainhash.Hash{filterHash} + serialized := serializeHeaderCommitmentsV1(commitments) + if len(serialized) != 0 { + err := commitmentsBucket.Put(node.hash[:], serialized) + if err != nil { + return err + } + } + + // Update in-progress tracking height for resume. + var serializedProgress [4]byte + byteOrder.PutUint32(serializedProgress[:], node.height) + err = meta.Put(hdrCmtProgressKeyName, serializedProgress[:]) + if err != nil { + return err + } + + numStored++ + } + + return nil + }() + isFullyDone := err == nil + if (isFullyDone || logProgress) && numStored > 0 { + totalStored += numStored + log.Infof("Stored %d entries (%d total)", numStored, totalStored) + } + // Remove in-progress tracking key when the upgrade is done. + if isFullyDone { + _ = meta.Delete(hdrCmtProgressKeyName) + } + return isFullyDone, err + } + + // Store the commitments in batches for the reasons mentioned above. + if err := batchedUpdate(ctx, db, doBatch); err != nil { + return err + } + + elapsed := time.Since(start).Round(time.Millisecond) + log.Infof("Done storing header commitments. Total entries: %d in %v", + totalStored, elapsed) + return nil +} + +// upgradeToVersion13 upgrades a version 12 blockchain database to version 13. +// This entails populating the header commitments journal for all blocks after +// the point that header commitments activated and also have the block data +// and associated filter data available. +func upgradeToVersion13(ctx context.Context, db database.DB, chainParams *chaincfg.Params, dbInfo *databaseInfo) error { + if interruptRequested(ctx) { + return errInterruptRequested + } + + log.Info("Upgrading database to version 13...") + start := time.Now() + + // Store header commitments for all applicable blocks. + err := initializeHeaderCmts(ctx, db, chainParams) + if err != nil { + return err + } + + // Update and persist the overall block database version. + err = db.Update(func(dbTx database.Tx) error { + dbInfo.version = 13 + return dbPutDatabaseInfo(dbTx, dbInfo) + }) + if err != nil { + return err + } + + elapsed := time.Since(start).Round(time.Millisecond) + log.Infof("Done upgrading database in %v.", elapsed) + return nil +} + // separateUtxoDatabase moves the UTXO set and state from the block database to // the UTXO database. func separateUtxoDatabase(ctx context.Context, db database.DB, @@ -5427,6 +5858,16 @@ func upgradeDB(ctx context.Context, db database.DB, chainParams *chaincfg.Params } } + // Update to a version 13 database if needed. This entails populating the + // header commitments journal for all blocks after the point that header + // commitments activated and also have the block data and associated filter + // data available. + if dbInfo.version == 12 { + if err := upgradeToVersion13(ctx, db, chainParams, dbInfo); err != nil { + return err + } + } + return nil }