multi: Introduce UTXO database.

This introduces a separate UTXO database that holds the UTXO database
info.  In a subsequent commit it will additionally hold the UTXO set and
state.

An overview of the changes is as follows:

- Introduce LoadUtxoDB to load (or create when needed) the new UTXO
  database on startup
- Add utxoDb and utxoDbInfo to BlockChain to hold the new UTXO database
- Initialize the UTXO database info in the UTXO database on startup
- Populate utxoDb for mock chains in tests

This is part of moving the UTXO set and state to a separate database.

blockchain/chain.go

@@ -137,6 +137,8 @@ type BlockChain struct {
 	deploymentVers      map[string]uint32
 	db                  database.DB
 	dbInfo              *databaseInfo
+	utxoDb              database.DB
+	utxoDbInfo          *utxoDatabaseInfo
 	chainParams         *chaincfg.Params
 	timeSource          MedianTimeSource
 	notifications       NotificationCallback
@@ -2102,11 +2104,17 @@ func (q *chainQueryerAdapter) PrevScripts(dbTx database.Tx, block *dcrutil.Block
 // Config is a descriptor which specifies the blockchain instance configuration.
 type Config struct {
 	// DB defines the database which houses the blocks and will be used to
-	// store all metadata created by this package such as the utxo set.
+	// store all metadata created by this package outside of the UTXO set, which
+	// is stored in a separate database.
 	//
 	// This field is required.
 	DB database.DB
 
+	// UtxoDB defines the database which houses the UTXO set.
+	//
+	// This field is required.
+	UtxoDB database.DB
+
 	// ChainParams identifies which chain parameters the chain is associated
 	// with.
 	//
@@ -2175,6 +2183,9 @@ func New(ctx context.Context, config *Config) (*BlockChain, error) {
 	if config.DB == nil {
 		return nil, AssertError("blockchain.New database is nil")
 	}
+	if config.UtxoDB == nil {
+		return nil, AssertError("blockchain.New UTXO database is nil")
+	}
 	if config.ChainParams == nil {
 		return nil, AssertError("blockchain.New chain parameters nil")
 	}
@@ -2215,6 +2226,7 @@ func New(ctx context.Context, config *Config) (*BlockChain, error) {
 		checkpointsByHeight:           checkpointsByHeight,
 		deploymentVers:                deploymentVers,
 		db:                            config.DB,
+		utxoDb:                        config.UtxoDB,
 		chainParams:                   params,
 		timeSource:                    config.TimeSource,
 		notifications:                 config.Notifications,
@@ -2243,6 +2255,12 @@ func New(ctx context.Context, config *Config) (*BlockChain, error) {
 		return nil, err
 	}
 
+	// Initialize the UTXO state.  This entails running any database migrations as
+	// necessary as well as initializing the UTXO cache.
+	if err := b.initUtxoState(ctx); err != nil {
+		return nil, err
+	}
+
 	// Initialize and catch up all of the currently active optional indexes
 	// as needed.
 	queryAdapter := chainQueryerAdapter{BlockChain: &b}
@@ -2257,6 +2275,9 @@ func New(ctx context.Context, config *Config) (*BlockChain, error) {
 		"%d, block index: %d", b.dbInfo.version, b.dbInfo.compVer,
 		b.dbInfo.bidxVer)
 
+	log.Infof("UTXO database version info: version: %d, compression: %d, utxo "+
+		"set: %d", b.utxoDbInfo.version, b.utxoDbInfo.compVer, b.utxoDbInfo.utxoVer)
+
 	b.index.RLock()
 	bestHdr := b.index.bestHeader
 	b.index.RUnlock()

blockchain/chainio.go

@@ -1362,6 +1362,14 @@ func (b *BlockChain) initChainState(ctx context.Context) error {
 		}
 	}
 
+	// Initialize the UTXO database info.  This must be initialized after the
+	// block database info is loaded, but before block database migrations are
+	// run, since setting the initial UTXO set version depends on the block
+	// database version as that is where it originally resided.
+	if err := b.initUtxoDbInfo(ctx); err != nil {
+		return err
+	}
+
 	// Upgrade the database as needed.
 	err = upgradeDB(ctx, b.db, b.chainParams, b.dbInfo)
 	if err != nil {
@@ -1463,14 +1471,7 @@ func (b *BlockChain) initChainState(ctx context.Context) error {
 	}
 
 	// Upgrade the database post block index load as needed.
-	err = upgradeDBPostBlockIndexLoad(ctx, b)
-	if err != nil {
-		return err
-	}
-
-	// Initialize the utxo cache to ensure that the state of the utxo set is
-	// caught up to the tip of the best chain.
-	return b.utxoCache.Initialize(b, tip)
+	return upgradeDBPostBlockIndexLoad(ctx, b)
 }
 
 // dbFetchBlockByNode uses an existing database transaction to retrieve the raw

blockchain/common_test.go

@@ -52,55 +52,79 @@ func isSupportedDbType(dbType string) bool {
 	return false
 }
 
-// chainSetup is used to create a new db and chain instance with the genesis
-// block already inserted.  In addition to the new chain instance, it returns
-// a teardown function the caller should invoke when done testing to clean up.
-func chainSetup(dbName string, params *chaincfg.Params) (*BlockChain, func(), error) {
-	if !isSupportedDbType(testDbType) {
-		return nil, nil, fmt.Errorf("unsupported db type %v", testDbType)
-	}
-
-	// Handle memory database specially since it doesn't need the disk
-	// specific handling.
+// createTestDatabase creates a test database with the provided database name
+// and database type for the given network.
+func createTestDatabase(dbName string, dbType string, net wire.CurrencyNet) (database.DB, func(), error) {
+	// Handle memory database specially since it doesn't need the disk specific
+	// handling.
 	var db database.DB
 	var teardown func()
-	if testDbType == "memdb" {
-		ndb, err := database.Create(testDbType)
+	if dbType == "memdb" {
+		ndb, err := database.Create(dbType)
 		if err != nil {
 			return nil, nil, fmt.Errorf("error creating db: %w", err)
 		}
 		db = ndb
 
-		// Setup a teardown function for cleaning up.  This function is
-		// returned to the caller to be invoked when it is done testing.
+		// Setup a teardown function for cleaning up.  This function is returned to
+		// the caller to be invoked when it is done testing.
 		teardown = func() {
 			db.Close()
 		}
 	} else {
-		// Create the directory for test database.
+		// Create the directory for the test database.
 		dbPath, err := ioutil.TempDir("", dbName)
 		if err != nil {
 			err := fmt.Errorf("unable to create test db path: %w",
 				err)
 			return nil, nil, err
 		}
 
-		// Create a new database to store the accepted blocks into.
-		ndb, err := database.Create(testDbType, dbPath, blockDataNet)
+		// Create the test database.
+		ndb, err := database.Create(dbType, dbPath, net)
 		if err != nil {
 			os.RemoveAll(dbPath)
 			return nil, nil, fmt.Errorf("error creating db: %w", err)
 		}
 		db = ndb
 
-		// Setup a teardown function for cleaning up.  This function is
-		// returned to the caller to be invoked when it is done testing.
+		// Setup a teardown function for cleaning up.  This function is returned to
+		// the caller to be invoked when it is done testing.
 		teardown = func() {
 			db.Close()
 			os.RemoveAll(dbPath)
 		}
 	}
 
+	return db, teardown, nil
+}
+
+// chainSetup is used to create a new db and chain instance with the genesis
+// block already inserted.  In addition to the new chain instance, it returns
+// a teardown function the caller should invoke when done testing to clean up.
+func chainSetup(dbName string, params *chaincfg.Params) (*BlockChain, func(), error) {
+	if !isSupportedDbType(testDbType) {
+		return nil, nil, fmt.Errorf("unsupported db type %v", testDbType)
+	}
+
+	// Create a test block database.
+	db, teardownDb, err := createTestDatabase(dbName, testDbType, blockDataNet)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Create a test UTXO database.
+	utxoDb, teardownUtxoDb, err := createTestDatabase(dbName+"_utxo", testDbType,
+		blockDataNet)
+	if err != nil {
+		teardownDb()
+		return nil, nil, err
+	}
+	teardown := func() {
+		teardownUtxoDb()
+		teardownDb()
+	}
+
 	// Copy the chain params to ensure any modifications the tests do to
 	// the chain parameters do not affect the global instance.
 	paramsCopy := *params
@@ -115,6 +139,7 @@ func chainSetup(dbName string, params *chaincfg.Params) (*BlockChain, func(), er
 	chain, err := New(context.Background(),
 		&Config{
 			DB:          db,
+			UtxoDB:      utxoDb,
 			ChainParams: &paramsCopy,
 			TimeSource:  NewMedianTime(),
 			SigCache:    sigCache,

blockchain/fullblocks_test.go

@@ -45,55 +45,79 @@ func isSupportedDbType(dbType string) bool {
 	return false
 }
 
-// chainSetup is used to create a new db and chain instance with the genesis
-// block already inserted.  In addition to the new chain instance, it returns
-// a teardown function the caller should invoke when done testing to clean up.
-func chainSetup(dbName string, params *chaincfg.Params) (*blockchain.BlockChain, func(), error) {
-	if !isSupportedDbType(testDbType) {
-		return nil, nil, fmt.Errorf("unsupported db type %v", testDbType)
-	}
-
-	// Handle memory database specially since it doesn't need the disk
-	// specific handling.
+// createTestDatabase creates a test database with the provided database name
+// and database type for the given network.
+func createTestDatabase(dbName string, dbType string, net wire.CurrencyNet) (database.DB, func(), error) {
+	// Handle memory database specially since it doesn't need the disk specific
+	// handling.
 	var db database.DB
 	var teardown func()
-	if testDbType == "memdb" {
-		ndb, err := database.Create(testDbType)
+	if dbType == "memdb" {
+		ndb, err := database.Create(dbType)
 		if err != nil {
 			return nil, nil, fmt.Errorf("error creating db: %w", err)
 		}
 		db = ndb
 
-		// Setup a teardown function for cleaning up.  This function is
-		// returned to the caller to be invoked when it is done testing.
+		// Setup a teardown function for cleaning up.  This function is returned to
+		// the caller to be invoked when it is done testing.
 		teardown = func() {
 			db.Close()
 		}
 	} else {
-		// Create the directory for test database.
+		// Create the directory for the test database.
 		dbPath, err := ioutil.TempDir("", dbName)
 		if err != nil {
 			err := fmt.Errorf("unable to create test db path: %w",
 				err)
 			return nil, nil, err
 		}
 
-		// Create a new database to store the accepted blocks into.
-		ndb, err := database.Create(testDbType, dbPath, blockDataNet)
+		// Create the test database.
+		ndb, err := database.Create(dbType, dbPath, net)
 		if err != nil {
 			os.RemoveAll(dbPath)
 			return nil, nil, fmt.Errorf("error creating db: %w", err)
 		}
 		db = ndb
 
-		// Setup a teardown function for cleaning up.  This function is
-		// returned to the caller to be invoked when it is done testing.
+		// Setup a teardown function for cleaning up.  This function is returned to
+		// the caller to be invoked when it is done testing.
 		teardown = func() {
 			db.Close()
 			os.RemoveAll(dbPath)
 		}
 	}
 
+	return db, teardown, nil
+}
+
+// chainSetup is used to create a new db and chain instance with the genesis
+// block already inserted.  In addition to the new chain instance, it returns
+// a teardown function the caller should invoke when done testing to clean up.
+func chainSetup(dbName string, params *chaincfg.Params) (*blockchain.BlockChain, func(), error) {
+	if !isSupportedDbType(testDbType) {
+		return nil, nil, fmt.Errorf("unsupported db type %v", testDbType)
+	}
+
+	// Create a test block database.
+	db, teardownDb, err := createTestDatabase(dbName, testDbType, blockDataNet)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Create a test UTXO database.
+	utxoDb, teardownUtxoDb, err := createTestDatabase(dbName+"_utxo", testDbType,
+		blockDataNet)
+	if err != nil {
+		teardownDb()
+		return nil, nil, err
+	}
+	teardown := func() {
+		teardownUtxoDb()
+		teardownDb()
+	}
+
 	// Copy the chain params to ensure any modifications the tests do to
 	// the chain parameters do not affect the global instance.
 	paramsCopy := *params
@@ -108,6 +132,7 @@ func chainSetup(dbName string, params *chaincfg.Params) (*blockchain.BlockChain,
 	chain, err := blockchain.New(context.Background(),
 		&blockchain.Config{
 			DB:          db,
+			UtxoDB:      utxoDb,
 			ChainParams: &paramsCopy,
 			TimeSource:  blockchain.NewMedianTime(),
 			SigCache:    sigCache,