From e10d21fa83c28aef78d536a54745c1212acb8126 Mon Sep 17 00:00:00 2001 From: Nils-Erik Frantzell Date: Fri, 8 Nov 2019 21:48:53 -0800 Subject: [PATCH 1/9] Add block size in bytes to BlockDetails structure Also expose this to RPC callers --- libethcore/Common.cpp | 4 +-- libethereum/Block.cpp | 8 ++--- libethereum/BlockChain.cpp | 47 ++++++++++++++++------------ libethereum/BlockDetails.cpp | 9 +++--- libethereum/BlockDetails.h | 20 +++++++----- libethereum/ClientBase.cpp | 8 +++-- libethereum/EthereumCapability.cpp | 2 +- libweb3jsonrpc/Eth.cpp | 50 +++++++++++++++++++++--------- 8 files changed, 92 insertions(+), 56 deletions(-) diff --git a/libethcore/Common.cpp b/libethcore/Common.cpp index d4f2e7dd608..b26e648de55 100644 --- a/libethcore/Common.cpp +++ b/libethcore/Common.cpp @@ -23,11 +23,11 @@ namespace eth const unsigned c_protocolVersion = 63; #if ETH_FATDB -const unsigned c_databaseMinorVersion = 3; +const unsigned c_databaseMinorVersion = 4; const unsigned c_databaseBaseVersion = 9; const unsigned c_databaseVersionModifier = 1; #else -const unsigned c_databaseMinorVersion = 2; +const unsigned c_databaseMinorVersion = 3; const unsigned c_databaseBaseVersion = 9; const unsigned c_databaseVersionModifier = 0; #endif diff --git a/libethereum/Block.cpp b/libethereum/Block.cpp index d4f0b48b4be..fbc686d20d2 100644 --- a/libethereum/Block.cpp +++ b/libethereum/Block.cpp @@ -574,8 +574,8 @@ u256 Block::enact(VerifiedBlockRef const& _block, BlockChain const& _bc) // cB.p^6 -----------/ 6 // cB.p^7 -------------/ // cB.p^8 - auto expectedUncleParent = _bc.details(m_currentBlock.parentHash()).parent; - for (unsigned i = 1; i < depth; expectedUncleParent = _bc.details(expectedUncleParent).parent, ++i) {} + auto expectedUncleParent = _bc.details(m_currentBlock.parentHash()).parentHash; + for (unsigned i = 1; i < depth; expectedUncleParent = _bc.details(expectedUncleParent).parentHash, ++i) {} if (expectedUncleParent != uncleParent.hash()) { UncleParentNotInChain ex; @@ -720,9 +720,9 @@ void Block::commitToSeal(BlockChain const& _bc, bytes const& _extraData) << ", parent = " << m_previousBlock.parentHash(); h256Hash excluded = _bc.allKinFrom(m_currentBlock.parentHash(), 6); auto p = m_previousBlock.parentHash(); - for (unsigned gen = 0; gen < 6 && p != _bc.genesisHash() && unclesCount < 2; ++gen, p = _bc.details(p).parent) + for (unsigned gen = 0; gen < 6 && p != _bc.genesisHash() && unclesCount < 2; ++gen, p = _bc.details(p).parentHash) { - auto us = _bc.details(p).children; + auto us = _bc.details(p).childHashes; assert(us.size() >= 1); // must be at least 1 child of our grandparent - it's our own parent! for (auto const& u: us) if (!excluded.count(u)) // ignore any uncles/mainline blocks that we know about. diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index 80390015c63..c34a7dfc819 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -229,7 +229,6 @@ bool BlockChain::open(fs::path const& _path, WithExisting _we) LOG(m_loggerInfo) << "Version from " << extrasSubPathMinor << " (" << lastMinor << ") != Aleth's version (" << c_databaseMinorVersion << ")"; writeMinorVersion = true; - lastMinor = (unsigned)RLP(minorVersionBytes); } } else @@ -278,9 +277,12 @@ bool BlockChain::open(fs::path const& _path, WithExisting _we) if (_we != WithExisting::Verify && !rebuildNeeded && !details(m_genesisHash)) { - BlockHeader gb(m_params.genesisBlock()); + bytes const genesisBlockBytes = m_params.genesisBlock(); + BlockHeader gb(genesisBlockBytes); // Insert details of genesis block. - m_details[m_genesisHash] = BlockDetails(0, gb.difficulty(), h256(), {}); + m_details[m_genesisHash] = + BlockDetails{0 /* number */, gb.difficulty(), h256{} /* parent */, {} /* children */, + static_cast(genesisBlockBytes.size())}; auto r = m_details[m_genesisHash].rlp(); m_extrasDB->insert(toSlice(m_genesisHash, ExtraDetails), (db::Slice)dev::ref(r)); assert(isKnown(gb.hash())); @@ -289,8 +291,9 @@ bool BlockChain::open(fs::path const& _path, WithExisting _we) // TODO: Implement ability to rebuild details map from DB. auto const l = m_extrasDB->lookup(db::Slice("best")); m_lastBlockHash = l.empty() ? m_genesisHash : h256(l, h256::FromBinary); - - m_lastBlockNumber = number(m_lastBlockHash); + // We need to retrieve the block number from the blocks database rather than from the extras + // database because the extras database format may have changed + m_lastBlockNumber = info(m_lastBlockHash).number(); ctrace << "Opened blockchain DB. Latest: " << currentHash() << (!rebuildNeeded ? "(rebuild not needed)" : "*** REBUILD NEEDED ***"); @@ -384,12 +387,12 @@ void BlockChain::rebuild(fs::path const& _path, std::function(s.blockData().size())}; m_details[m_genesisHash] = genesisDetails; auto const genesisDetailsRlp = genesisDetails.rlp(); m_extrasDB->insert( toSlice(m_genesisHash, ExtraDetails), (db::Slice)dev::ref(genesisDetailsRlp)); - LOG(m_loggerInfo) << "Rebuilding the extras and state databases by reimporting blocks 0 -> " << originalNumber << ", this will probably take a while"; h256 lastHash = m_lastBlockHash; @@ -635,8 +638,8 @@ void BlockChain::insert(VerifiedBlockRef _block, bytesConstRef _receipts, bool _ details(_block.info.parentHash()); DEV_WRITE_GUARDED(x_details) { - if (!dev::contains(m_details[_block.info.parentHash()].children, _block.info.hash())) - m_details[_block.info.parentHash()].children.push_back(_block.info.hash()); + if (!dev::contains(m_details[_block.info.parentHash()].childHashes, _block.info.hash())) + m_details[_block.info.parentHash()].childHashes.push_back(_block.info.hash()); } blocksWriteBatch->insert(toSlice(_block.info.hash()), db::Slice(_block.block)); @@ -644,7 +647,9 @@ void BlockChain::insert(VerifiedBlockRef _block, bytesConstRef _receipts, bool _ extrasWriteBatch->insert(toSlice(_block.info.parentHash(), ExtraDetails), (db::Slice)dev::ref(m_details[_block.info.parentHash()].rlp())); - BlockDetails bd((unsigned)pd.number + 1, pd.totalDifficulty + _block.info.difficulty(), _block.info.parentHash(), {}); + BlockDetails bd{static_cast(pd.number + 1), + pd.totalDifficulty + _block.info.difficulty(), _block.info.parentHash(), {} /* children */, + static_cast(_block.block.size())}; extrasWriteBatch->insert( toSlice(_block.info.hash(), ExtraDetails), (db::Slice)dev::ref(bd.rlp())); extrasWriteBatch->insert( @@ -804,7 +809,7 @@ ImportRoute BlockChain::insertBlockAndExtras(VerifiedBlockRef const& _block, byt // done here. details(_block.info.parentHash()); DEV_WRITE_GUARDED(x_details) - m_details[_block.info.parentHash()].children.push_back(_block.info.hash()); + m_details[_block.info.parentHash()].childHashes.push_back(_block.info.hash()); _performanceLogger.onStageFinished("collation"); @@ -813,7 +818,9 @@ ImportRoute BlockChain::insertBlockAndExtras(VerifiedBlockRef const& _block, byt extrasWriteBatch->insert(toSlice(_block.info.parentHash(), ExtraDetails), (db::Slice)dev::ref(m_details[_block.info.parentHash()].rlp())); - BlockDetails const details((unsigned)_block.info.number(), _totalDifficulty, _block.info.parentHash(), {}); + BlockDetails const details{static_cast(_block.info.number()), _totalDifficulty, + _block.info.parentHash(), {} /* children */, + static_cast(_block.block.size())}; extrasWriteBatch->insert( toSlice(_block.info.hash(), ExtraDetails), (db::Slice)dev::ref(details.rlp())); @@ -912,7 +919,7 @@ ImportRoute BlockChain::insertBlockAndExtras(VerifiedBlockRef const& _block, byt LOG(m_logger) << " Imported and best " << _totalDifficulty << " (#" << _block.info.number() << "). Has " - << (details(_block.info.parentHash()).children.size() - 1) + << (details(_block.info.parentHash()).childHashes.size() - 1) << " siblings. Route: " << route; } else @@ -1128,7 +1135,7 @@ tuple BlockChain::treeRoute(h256 const& _from, h256 const { if (_pre) ret.push_back(from); - from = details(from).parent; + from = details(from).parentHash; fn--; } @@ -1138,10 +1145,10 @@ tuple BlockChain::treeRoute(h256 const& _from, h256 const { if (_post) back.push_back(to); - to = details(to).parent; + to = details(to).parentHash; tn--; } - for (;; from = details(from).parent, to = details(to).parent) + for (;; from = details(from).parentHash, to = details(to).parentHash) { if (_pre && (from != to || _common)) ret.push_back(from); @@ -1279,13 +1286,13 @@ void BlockChain::checkConsistency() { h256 h((byte const*)_key.data(), h256::ConstructFromPointer); auto dh = details(h); - auto p = dh.parent; + auto p = dh.parentHash; if (p != h256() && p != m_genesisHash) // TODO: for some reason the genesis details // with the children get squished. not sure // why. { auto dp = details(p); - if (asserts(contains(dp.children, h))) + if (asserts(contains(dp.childHashes, h))) cnote << "Apparently the database is corrupt. Not much we can do at this " "stage..."; if (assertsEqual(dp.number, dh.number - 1)) @@ -1387,9 +1394,9 @@ h256Hash BlockChain::allKinFrom(h256 const& _parent, unsigned _generations) cons h256 p = _parent; h256Hash ret = { p }; // p and (details(p).parent: i == 5) is likely to be overkill, but can't hurt to be cautious. - for (unsigned i = 0; i < _generations && p != m_genesisHash; ++i, p = details(p).parent) + for (unsigned i = 0; i < _generations && p != m_genesisHash; ++i, p = details(p).parentHash) { - ret.insert(details(p).parent); + ret.insert(details(p).parentHash); auto b = block(p); for (auto i: RLP(b)[2]) ret.insert(sha3(i.data())); diff --git a/libethereum/BlockDetails.cpp b/libethereum/BlockDetails.cpp index 0d7e7c5634a..f26a29fb98a 100644 --- a/libethereum/BlockDetails.cpp +++ b/libethereum/BlockDetails.cpp @@ -14,14 +14,13 @@ BlockDetails::BlockDetails(RLP const& _r) { number = _r[0].toInt(); totalDifficulty = _r[1].toInt(); - parent = _r[2].toHash(); - children = _r[3].toVector(); + parentHash = _r[2].toHash(); + childHashes = _r[3].toVector(); size = _r.size(); + blockSizeBytes = _r[4].toInt(); } bytes BlockDetails::rlp() const { - auto ret = rlpList(number, totalDifficulty, parent, children); - size = ret.size(); - return ret; + return rlpList(number, totalDifficulty, parentHash, childHashes, blockSizeBytes); } diff --git a/libethereum/BlockDetails.h b/libethereum/BlockDetails.h index 8c5f8e6f387..b5e85d12795 100644 --- a/libethereum/BlockDetails.h +++ b/libethereum/BlockDetails.h @@ -17,15 +17,16 @@ namespace eth // TODO: OPTIMISE: constructors take bytes, RLP used only in necessary classes. -static const unsigned c_bloomIndexSize = 16; -static const unsigned c_bloomIndexLevels = 2; +constexpr unsigned c_bloomIndexSize = 16; +constexpr unsigned c_bloomIndexLevels = 2; -static const unsigned c_invalidNumber = (unsigned)-1; +constexpr unsigned c_invalidNumber = (unsigned)-1; struct BlockDetails { BlockDetails(): number(c_invalidNumber), totalDifficulty(Invalid256) {} - BlockDetails(unsigned _n, u256 _tD, h256 _p, h256s _c): number(_n), totalDifficulty(_tD), parent(_p), children(_c) {} + BlockDetails(unsigned _number, u256 _totalDifficulty, h256 _parentHash, h256s _childHashes, unsigned _blockSizeBytes) : number{ _number }, totalDifficulty{ _totalDifficulty }, parentHash{ _parentHash }, childHashes{ _childHashes + }, blockSizeBytes{_blockSizeBytes}{} BlockDetails(RLP const& _r); bytes rlp() const; @@ -34,10 +35,15 @@ struct BlockDetails unsigned number = c_invalidNumber; u256 totalDifficulty = Invalid256; - h256 parent; - h256s children; + h256 parentHash; + h256s childHashes; - mutable unsigned size; + // Size of the BlockDetails RLP (in bytes). Used for computing blockchain memory usage + // statistics. Field name must be 'size' as BlockChain::getHashSize depends on this + mutable unsigned size; + + // Size of the block RLP data in bytes + unsigned blockSizeBytes; }; struct BlockLogBlooms diff --git a/libethereum/ClientBase.cpp b/libethereum/ClientBase.cpp index 59181a18e19..48b6761df1d 100644 --- a/libethereum/ClientBase.cpp +++ b/libethereum/ClientBase.cpp @@ -417,9 +417,11 @@ BlockHeader ClientBase::pendingInfo() const BlockDetails ClientBase::pendingDetails() const { - auto pm = postSeal().info(); - auto li = Interface::blockDetails(LatestBlock); - return BlockDetails((unsigned)pm.number(), li.totalDifficulty + pm.difficulty(), pm.parentHash(), h256s{}); + auto pendingHeader = postSeal().info(); + auto latestDetails = Interface::blockDetails(LatestBlock); + return BlockDetails{(unsigned)pendingHeader.number(), + latestDetails.totalDifficulty + pendingHeader.difficulty(), pendingHeader.parentHash(), + h256s{} /* children */, static_cast(postSeal().blockData().size())}; } Addresses ClientBase::addresses(BlockNumber _block) const diff --git a/libethereum/EthereumCapability.cpp b/libethereum/EthereumCapability.cpp index ce768243dde..72539e9630e 100644 --- a/libethereum/EthereumCapability.cpp +++ b/libethereum/EthereumCapability.cpp @@ -264,7 +264,7 @@ class EthereumHostData : public EthereumHostDataFace if (details.number < limitBlock) // stop using parent hash traversal, fallback to using block numbers break; - _h = details.parent; + _h = details.parentHash; --_step; } diff --git a/libweb3jsonrpc/Eth.cpp b/libweb3jsonrpc/Eth.cpp index 80c76571053..62ec54e835a 100644 --- a/libweb3jsonrpc/Eth.cpp +++ b/libweb3jsonrpc/Eth.cpp @@ -345,17 +345,28 @@ Json::Value Eth::eth_getBlockByHash(string const& _blockHash, bool _includeTrans { try { - h256 h = jsToFixed<32>(_blockHash); + h256 const h = jsToFixed<32>(_blockHash); if (!client()->isKnown(h)) return Json::Value(Json::nullValue); - if (_includeTransactions) - return toJson(client()->blockInfo(h), client()->blockDetails(h), client()->uncleHashes(h), client()->transactions(h), client()->sealEngine()); - else - return toJson(client()->blockInfo(h), client()->blockDetails(h), client()->uncleHashes(h), client()->transactionHashes(h), client()->sealEngine()); - } - catch (...) - { + Json::Value ret; + auto const blockDetails = client()->blockDetails(h); + if (_includeTransactions) + ret = toJson(client()->blockInfo(h), blockDetails, + client()->uncleHashes(h), client()->transactions(h), client()->sealEngine()); + else + ret = toJson(client()->blockInfo(h), blockDetails, + client()->uncleHashes(h), client()->transactionHashes(h), client()->sealEngine()); + + // We need to explicitly set the "size" field to the block size in bytes since the + // BlockDetails "size" field refers to something else (size of BlockDetails RLP) and cannot + // be changed + ret["size"] = toJS(blockDetails.blockSizeBytes); + + return ret; + } + catch (...) + { BOOST_THROW_EXCEPTION(JsonRpcException(Errors::ERROR_RPC_INVALID_PARAMS)); } } @@ -364,15 +375,26 @@ Json::Value Eth::eth_getBlockByNumber(string const& _blockNumber, bool _includeT { try { - BlockNumber h = jsToBlockNumber(_blockNumber); + BlockNumber const h = jsToBlockNumber(_blockNumber); if (!client()->isKnown(h)) return Json::Value(Json::nullValue); - if (_includeTransactions) - return toJson(client()->blockInfo(h), client()->blockDetails(h), client()->uncleHashes(h), client()->transactions(h), client()->sealEngine()); - else - return toJson(client()->blockInfo(h), client()->blockDetails(h), client()->uncleHashes(h), client()->transactionHashes(h), client()->sealEngine()); - } + Json::Value ret; + auto const blockDetails = client()->blockDetails(h); + if (_includeTransactions) + ret = toJson(client()->blockInfo(h), blockDetails, client()->uncleHashes(h), + client()->transactions(h), client()->sealEngine()); + else + ret = toJson(client()->blockInfo(h), blockDetails, client()->uncleHashes(h), + client()->transactionHashes(h), client()->sealEngine()); + + // We need to explicitly set the "size" field to the block size in bytes since the + // BlockDetails "size" field refers to something else (size of BlockDetails RLP) and cannot + // be changed + ret["size"] = toJS(blockDetails.blockSizeBytes); + + return ret; + } catch (...) { BOOST_THROW_EXCEPTION(JsonRpcException(Errors::ERROR_RPC_INVALID_PARAMS)); From ac319ac6b18090ad7d50aaaf38c3f7a6ab0db6b7 Mon Sep 17 00:00:00 2001 From: Nils-Erik Frantzell Date: Sat, 16 Nov 2019 22:14:38 -0800 Subject: [PATCH 2/9] Minor updates to rebuild logic Throw exception on rebuild failure, detect existence of temporary extras database and throw exception, only update database minor version file on successful rebuild --- libethcore/Exceptions.h | 2 + libethereum/BlockChain.cpp | 86 ++++++++++++++++++++++++-------------- 2 files changed, 57 insertions(+), 31 deletions(-) diff --git a/libethcore/Exceptions.h b/libethcore/Exceptions.h index 81eac09ed5d..6311f8ebbeb 100644 --- a/libethcore/Exceptions.h +++ b/libethcore/Exceptions.h @@ -76,6 +76,8 @@ DEV_SIMPLE_EXCEPTION(UnknownError); DEV_SIMPLE_EXCEPTION(InvalidDatabaseKind); DEV_SIMPLE_EXCEPTION(DatabaseAlreadyOpen); DEV_SIMPLE_EXCEPTION(DatabaseCorruption); +DEV_SIMPLE_EXCEPTION(DatabaseExists); +DEV_SIMPLE_EXCEPTION(DatabaseRebuildFailed); DEV_SIMPLE_EXCEPTION(DAGCreationFailure); DEV_SIMPLE_EXCEPTION(DAGComputeFailure); diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index c34a7dfc819..e078a28d5a4 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -206,10 +206,8 @@ bool BlockChain::open(fs::path const& _path, WithExisting _we) { if (_we == WithExisting::Kill) { - cnote << "Killing blockchain (" << chainSubPathBlocks << ") & extras (" - << extrasSubPathExtras << ") databases (WithExisting::Kill)."; - fs::remove_all(chainSubPathBlocks); - fs::remove_all(extrasSubPathExtras); + LOG(m_loggerInfo) << "Deleting all databases. This will require a resync from genesis."; + fs::remove_all(chainPath); } fs::create_directories(extrasPath); @@ -217,7 +215,6 @@ bool BlockChain::open(fs::path const& _path, WithExisting _we) auto const extrasSubPathMinor = extrasPath / fs::path("minor"); bytes const minorVersionBytes = contents(extrasSubPathMinor); - bool writeMinorVersion = false; if (!minorVersionBytes.empty()) { DEV_IGNORE_EXCEPTIONS(lastMinor = (unsigned)RLP(minorVersionBytes)); @@ -228,13 +225,15 @@ bool BlockChain::open(fs::path const& _path, WithExisting _we) "databases will be rebuilt."; LOG(m_loggerInfo) << "Version from " << extrasSubPathMinor << " (" << lastMinor << ") != Aleth's version (" << c_databaseMinorVersion << ")"; - writeMinorVersion = true; } } else - writeMinorVersion = true; - if (writeMinorVersion) + { + LOG(m_loggerDetail) << "Creating database minor version file: " << extrasSubPathMinor + << " (current database minor version: " << c_databaseMinorVersion + << ")"; writeFile(extrasSubPathMinor, rlp(c_databaseMinorVersion)); + } } try @@ -246,32 +245,35 @@ bool BlockChain::open(fs::path const& _path, WithExisting _we) { // Determine which database open call failed auto const dbPath = !m_blocksDB.get() ? chainSubPathBlocks : extrasSubPathExtras; - cerror << "Error opening database: " << dbPath; + LOG(m_loggerError) << "Error opening database: " << dbPath; if (db::isDiskDatabase()) { db::DatabaseStatus const dbStatus = *boost::get_error_info(ex); if (fs::space(path).available < 1024) { - cerror << "Not enough available space found on hard drive. Please free some up and " - "re-run."; + LOG(m_loggerError) + << "Not enough available space found on hard drive. Please free some up and " + "re-run."; BOOST_THROW_EXCEPTION(NotEnoughAvailableSpace()); } else if (dbStatus == db::DatabaseStatus::Corruption) { - cerror << "Database corruption detected. Please see the exception for corruption " - "details. Exception: " + LOG(m_loggerError) + << "Database corruption detected. Please see the exception for corruption " + "details. Exception: " << ex.what(); BOOST_THROW_EXCEPTION(DatabaseCorruption()); } else if (dbStatus == db::DatabaseStatus::IOError) { - cerror << "Database already open. You appear to have another instance of Aleth running."; + LOG(m_loggerError) << "Database already open. You appear to have another instance " + "of Aleth running."; BOOST_THROW_EXCEPTION(DatabaseAlreadyOpen()); } } - - cerror << "Unknown error occurred. Exception details: " << ex.what(); + + LOG(m_loggerError) << "Unknown error occurred. Exception details: " << ex.what(); throw; } @@ -295,8 +297,8 @@ bool BlockChain::open(fs::path const& _path, WithExisting _we) // database because the extras database format may have changed m_lastBlockNumber = info(m_lastBlockHash).number(); - ctrace << "Opened blockchain DB. Latest: " << currentHash() - << (!rebuildNeeded ? "(rebuild not needed)" : "*** REBUILD NEEDED ***"); + LOG(m_loggerInfo) << "Opened blockchain DB. Latest: " << currentHash() + << (!rebuildNeeded ? "(rebuild not needed)" : "*** REBUILD NEEDED ***"); return rebuildNeeded; } @@ -360,8 +362,18 @@ void BlockChain::rebuild(fs::path const& _path, std::function oldExtrasDB(db::DBFactory::create(extrasSubPathOldExtras)); m_extrasDB = db::DBFactory::create(extrasSubPathExtras); @@ -387,16 +399,19 @@ void BlockChain::rebuild(fs::path const& _path, std::function(s.blockData().size())}; + auto const genesisDetails = + BlockDetails{0 /* block number */, s.info().difficulty(), h256{} /* parent */, + {} /* children */, static_cast(m_params.genesisBlock().size())}; m_details[m_genesisHash] = genesisDetails; - auto const genesisDetailsRlp = genesisDetails.rlp(); m_extrasDB->insert( - toSlice(m_genesisHash, ExtraDetails), (db::Slice)dev::ref(genesisDetailsRlp)); + toSlice(m_genesisHash, ExtraDetails), (db::Slice)dev::ref(genesisDetails.rlp())); + LOG(m_loggerInfo) << "Rebuilding the extras and state databases by reimporting blocks 0 -> " << originalNumber << ", this will probably take a while"; h256 lastHash = m_lastBlockHash; Timer t; + bool rebuildFailed = false; + string exceptionInfo; for (unsigned d = 1; d <= originalNumber; ++d) { if (!(d % 1000)) @@ -425,21 +440,30 @@ void BlockChain::rebuild(fs::path const& _path, std::function Date: Sun, 17 Nov 2019 22:18:31 -0800 Subject: [PATCH 3/9] Address PR feedback Remove individual directories in BlockChain::open on WithExisting::Kill and save genesis block details RLP in local variable before inserting into extras db. Untabify BlockDetails and pass ctor args by const& Cleanup web3.eth.getBlock* calls Make BlockDetails::blockSizeBytes a size_t instead of unsigned Minor formatting/logging changes (e.g. untabify BlockDetails files) --- libethereum/Block.cpp | 3 +- libethereum/BlockChain.cpp | 30 ++++++------ libethereum/BlockDetails.cpp | 12 ++--- libethereum/BlockDetails.h | 90 +++++++++++++++++++---------------- libweb3jsonrpc/Eth.cpp | 36 ++++++-------- libweb3jsonrpc/JsonHelper.cpp | 4 +- 6 files changed, 88 insertions(+), 87 deletions(-) diff --git a/libethereum/Block.cpp b/libethereum/Block.cpp index fbc686d20d2..7f59564f34d 100644 --- a/libethereum/Block.cpp +++ b/libethereum/Block.cpp @@ -720,7 +720,8 @@ void Block::commitToSeal(BlockChain const& _bc, bytes const& _extraData) << ", parent = " << m_previousBlock.parentHash(); h256Hash excluded = _bc.allKinFrom(m_currentBlock.parentHash(), 6); auto p = m_previousBlock.parentHash(); - for (unsigned gen = 0; gen < 6 && p != _bc.genesisHash() && unclesCount < 2; ++gen, p = _bc.details(p).parentHash) + for (unsigned gen = 0; gen < 6 && p != _bc.genesisHash() && unclesCount < 2; + ++gen, p = _bc.details(p).parentHash) { auto us = _bc.details(p).childHashes; assert(us.size() >= 1); // must be at least 1 child of our grandparent - it's our own parent! diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index e078a28d5a4..18bedad6b66 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -206,8 +206,11 @@ bool BlockChain::open(fs::path const& _path, WithExisting _we) { if (_we == WithExisting::Kill) { - LOG(m_loggerInfo) << "Deleting all databases. This will require a resync from genesis."; - fs::remove_all(chainPath); + LOG(m_loggerInfo) + << "Deleting chain databases. This will require a resync from genesis."; + fs::remove_all(chainSubPathBlocks); + fs::remove_all(extrasSubPathExtras); + fs::remove_all(extrasPath / fs::path("extras.old")); } fs::create_directories(extrasPath); @@ -230,8 +233,7 @@ bool BlockChain::open(fs::path const& _path, WithExisting _we) else { LOG(m_loggerDetail) << "Creating database minor version file: " << extrasSubPathMinor - << " (current database minor version: " << c_databaseMinorVersion - << ")"; + << " (minor version: " << c_databaseMinorVersion << ")"; writeFile(extrasSubPathMinor, rlp(c_databaseMinorVersion)); } } @@ -280,11 +282,10 @@ bool BlockChain::open(fs::path const& _path, WithExisting _we) if (_we != WithExisting::Verify && !rebuildNeeded && !details(m_genesisHash)) { bytes const genesisBlockBytes = m_params.genesisBlock(); - BlockHeader gb(genesisBlockBytes); + BlockHeader gb{genesisBlockBytes}; // Insert details of genesis block. - m_details[m_genesisHash] = - BlockDetails{0 /* number */, gb.difficulty(), h256{} /* parent */, {} /* children */, - static_cast(genesisBlockBytes.size())}; + m_details[m_genesisHash] = BlockDetails{0 /* number */, gb.difficulty(), + h256{} /* parent */, {} /* children */, genesisBlockBytes.size()}; auto r = m_details[m_genesisHash].rlp(); m_extrasDB->insert(toSlice(m_genesisHash, ExtraDetails), (db::Slice)dev::ref(r)); assert(isKnown(gb.hash())); @@ -399,12 +400,12 @@ void BlockChain::rebuild(fs::path const& _path, std::function(m_params.genesisBlock().size())}; + auto const genesisDetails = BlockDetails{0 /* block number */, s.info().difficulty(), + h256{} /* parent */, {} /* children */, m_params.genesisBlock().size()}; m_details[m_genesisHash] = genesisDetails; + auto const genesisDetailsRlp = genesisDetails.rlp(); m_extrasDB->insert( - toSlice(m_genesisHash, ExtraDetails), (db::Slice)dev::ref(genesisDetails.rlp())); + toSlice(m_genesisHash, ExtraDetails), (db::Slice)dev::ref(genesisDetailsRlp)); LOG(m_loggerInfo) << "Rebuilding the extras and state databases by reimporting blocks 0 -> " << originalNumber << ", this will probably take a while"; @@ -673,7 +674,7 @@ void BlockChain::insert(VerifiedBlockRef _block, bytesConstRef _receipts, bool _ BlockDetails bd{static_cast(pd.number + 1), pd.totalDifficulty + _block.info.difficulty(), _block.info.parentHash(), {} /* children */, - static_cast(_block.block.size())}; + _block.block.size()}; extrasWriteBatch->insert( toSlice(_block.info.hash(), ExtraDetails), (db::Slice)dev::ref(bd.rlp())); extrasWriteBatch->insert( @@ -843,8 +844,7 @@ ImportRoute BlockChain::insertBlockAndExtras(VerifiedBlockRef const& _block, byt (db::Slice)dev::ref(m_details[_block.info.parentHash()].rlp())); BlockDetails const details{static_cast(_block.info.number()), _totalDifficulty, - _block.info.parentHash(), {} /* children */, - static_cast(_block.block.size())}; + _block.info.parentHash(), {} /* children */, _block.block.size()}; extrasWriteBatch->insert( toSlice(_block.info.hash(), ExtraDetails), (db::Slice)dev::ref(details.rlp())); diff --git a/libethereum/BlockDetails.cpp b/libethereum/BlockDetails.cpp index f26a29fb98a..660f024abc8 100644 --- a/libethereum/BlockDetails.cpp +++ b/libethereum/BlockDetails.cpp @@ -12,12 +12,12 @@ using namespace dev::eth; BlockDetails::BlockDetails(RLP const& _r) { - number = _r[0].toInt(); - totalDifficulty = _r[1].toInt(); - parentHash = _r[2].toHash(); - childHashes = _r[3].toVector(); - size = _r.size(); - blockSizeBytes = _r[4].toInt(); + number = _r[0].toInt(); + totalDifficulty = _r[1].toInt(); + parentHash = _r[2].toHash(); + childHashes = _r[3].toVector(); + size = _r.size(); + blockSizeBytes = _r[4].toInt(); } bytes BlockDetails::rlp() const diff --git a/libethereum/BlockDetails.h b/libethereum/BlockDetails.h index b5e85d12795..010a608b354 100644 --- a/libethereum/BlockDetails.h +++ b/libethereum/BlockDetails.h @@ -24,81 +24,87 @@ constexpr unsigned c_invalidNumber = (unsigned)-1; struct BlockDetails { - BlockDetails(): number(c_invalidNumber), totalDifficulty(Invalid256) {} - BlockDetails(unsigned _number, u256 _totalDifficulty, h256 _parentHash, h256s _childHashes, unsigned _blockSizeBytes) : number{ _number }, totalDifficulty{ _totalDifficulty }, parentHash{ _parentHash }, childHashes{ _childHashes - }, blockSizeBytes{_blockSizeBytes}{} - BlockDetails(RLP const& _r); - bytes rlp() const; - - bool isNull() const { return number == c_invalidNumber; } - explicit operator bool() const { return !isNull(); } - - unsigned number = c_invalidNumber; - u256 totalDifficulty = Invalid256; - h256 parentHash; - h256s childHashes; + BlockDetails(): number(c_invalidNumber), totalDifficulty(Invalid256) {} + BlockDetails(unsigned _number, u256 const& _totalDifficulty, h256 const& _parentHash, h256s const& _childHashes, + size_t _blockSizeBytes) + : number{_number}, + totalDifficulty{_totalDifficulty}, + parentHash{_parentHash}, + childHashes{_childHashes}, + blockSizeBytes{_blockSizeBytes} + {} + BlockDetails(RLP const& _r); + bytes rlp() const; + + bool isNull() const { return number == c_invalidNumber; } + explicit operator bool() const { return !isNull(); } + + unsigned number = c_invalidNumber; + u256 totalDifficulty = Invalid256; + h256 parentHash; + h256s childHashes; // Size of the BlockDetails RLP (in bytes). Used for computing blockchain memory usage // statistics. Field name must be 'size' as BlockChain::getHashSize depends on this mutable unsigned size; // Size of the block RLP data in bytes - unsigned blockSizeBytes; + size_t blockSizeBytes; }; struct BlockLogBlooms { - BlockLogBlooms() {} - BlockLogBlooms(RLP const& _r) { blooms = _r.toVector(); size = _r.data().size(); } - bytes rlp() const { bytes r = dev::rlp(blooms); size = r.size(); return r; } + BlockLogBlooms() {} + BlockLogBlooms(RLP const& _r) { blooms = _r.toVector(); size = _r.data().size(); } + bytes rlp() const { bytes r = dev::rlp(blooms); size = r.size(); return r; } - LogBlooms blooms; - mutable unsigned size; + LogBlooms blooms; + mutable unsigned size; }; struct BlocksBlooms { - BlocksBlooms() {} - BlocksBlooms(RLP const& _r) { blooms = _r.toArray(); size = _r.data().size(); } - bytes rlp() const { bytes r = dev::rlp(blooms); size = r.size(); return r; } + BlocksBlooms() {} + BlocksBlooms(RLP const& _r) { blooms = _r.toArray(); size = _r.data().size(); } + bytes rlp() const { bytes r = dev::rlp(blooms); size = r.size(); return r; } - std::array blooms; - mutable unsigned size; + std::array blooms; + mutable unsigned size; }; struct BlockReceipts { - BlockReceipts() {} - BlockReceipts(RLP const& _r) { for (auto const& i: _r) receipts.emplace_back(i.data()); size = _r.data().size(); } - bytes rlp() const { RLPStream s(receipts.size()); for (TransactionReceipt const& i: receipts) i.streamRLP(s); size = s.out().size(); return s.out(); } + BlockReceipts() {} + BlockReceipts(RLP const& _r) { for (auto const& i: _r) receipts.emplace_back(i.data()); size = _r.data().size(); } + bytes rlp() const { RLPStream s(receipts.size()); for (TransactionReceipt const& i: receipts) i.streamRLP(s); size = s.out().size(); return s.out(); } - TransactionReceipts receipts; - mutable unsigned size = 0; + TransactionReceipts receipts; + mutable unsigned size = 0; }; struct BlockHash { - BlockHash() {} - BlockHash(h256 const& _h): value(_h) {} - BlockHash(RLP const& _r) { value = _r.toHash(); } - bytes rlp() const { return dev::rlp(value); } + BlockHash() {} + BlockHash(h256 const& _h): value(_h) {} + BlockHash(RLP const& _r) { value = _r.toHash(); } + bytes rlp() const { return dev::rlp(value); } - h256 value; - static const unsigned size = 65; + h256 value; + static const unsigned size = 65; }; struct TransactionAddress { - TransactionAddress() {} - TransactionAddress(RLP const& _rlp) { blockHash = _rlp[0].toHash(); index = _rlp[1].toInt(); } - bytes rlp() const { RLPStream s(2); s << blockHash << index; return s.out(); } + TransactionAddress() {} + TransactionAddress(RLP const& _rlp) { blockHash = _rlp[0].toHash(); index = _rlp[1].toInt(); } + bytes rlp() const { RLPStream s(2); s << blockHash << index; return s.out(); } - explicit operator bool() const { return !!blockHash; } + explicit operator bool() const { return !!blockHash; } - h256 blockHash; - unsigned index = 0; + h256 blockHash; + unsigned index = 0; - static const unsigned size = 67; + static const unsigned size = 67; }; using BlockDetailsHash = std::unordered_map; diff --git a/libweb3jsonrpc/Eth.cpp b/libweb3jsonrpc/Eth.cpp index 62ec54e835a..0a15ec0ee18 100644 --- a/libweb3jsonrpc/Eth.cpp +++ b/libweb3jsonrpc/Eth.cpp @@ -351,19 +351,16 @@ Json::Value Eth::eth_getBlockByHash(string const& _blockHash, bool _includeTrans Json::Value ret; auto const blockDetails = client()->blockDetails(h); + auto const blockHeader = client()->blockInfo(h); + auto const uncleHashes = client()->uncleHashes(h); + auto* sealEngine = client()->sealEngine(); if (_includeTransactions) - ret = toJson(client()->blockInfo(h), blockDetails, - client()->uncleHashes(h), client()->transactions(h), client()->sealEngine()); + ret = toJson( + blockHeader, blockDetails, uncleHashes, client()->transactions(h), sealEngine); else - ret = toJson(client()->blockInfo(h), blockDetails, - client()->uncleHashes(h), client()->transactionHashes(h), client()->sealEngine()); - - // We need to explicitly set the "size" field to the block size in bytes since the - // BlockDetails "size" field refers to something else (size of BlockDetails RLP) and cannot - // be changed - ret["size"] = toJS(blockDetails.blockSizeBytes); - - return ret; + ret = toJson( + blockHeader, blockDetails, uncleHashes, client()->transactionHashes(h), sealEngine); + return ret; } catch (...) { @@ -381,18 +378,15 @@ Json::Value Eth::eth_getBlockByNumber(string const& _blockNumber, bool _includeT Json::Value ret; auto const blockDetails = client()->blockDetails(h); + auto const blockHeader = client()->blockInfo(h); + auto const uncleHashes = client()->uncleHashes(h); + auto* sealEngine = client()->sealEngine(); if (_includeTransactions) - ret = toJson(client()->blockInfo(h), blockDetails, client()->uncleHashes(h), - client()->transactions(h), client()->sealEngine()); + ret = toJson( + blockHeader, blockDetails, uncleHashes, client()->transactions(h), sealEngine); else - ret = toJson(client()->blockInfo(h), blockDetails, client()->uncleHashes(h), - client()->transactionHashes(h), client()->sealEngine()); - - // We need to explicitly set the "size" field to the block size in bytes since the - // BlockDetails "size" field refers to something else (size of BlockDetails RLP) and cannot - // be changed - ret["size"] = toJS(blockDetails.blockSizeBytes); - + ret = toJson( + blockHeader, blockDetails, uncleHashes, client()->transactionHashes(h), sealEngine); return ret; } catch (...) diff --git a/libweb3jsonrpc/JsonHelper.cpp b/libweb3jsonrpc/JsonHelper.cpp index 18681a57ada..9e6f75147f8 100644 --- a/libweb3jsonrpc/JsonHelper.cpp +++ b/libweb3jsonrpc/JsonHelper.cpp @@ -125,7 +125,7 @@ Json::Value toJson(dev::eth::BlockHeader const& _bi, BlockDetails const& _bd, Un if (_bi) { res["totalDifficulty"] = toJS(_bd.totalDifficulty); - res["size"] = toJS(_bd.size); + res["size"] = toJS(_bd.blockSizeBytes); res["uncles"] = Json::Value(Json::arrayValue); for (h256 h: _us) res["uncles"].append(toJS(h)); @@ -142,7 +142,7 @@ Json::Value toJson(dev::eth::BlockHeader const& _bi, BlockDetails const& _bd, Un if (_bi) { res["totalDifficulty"] = toJS(_bd.totalDifficulty); - res["size"] = toJS(_bd.size); + res["size"] = toJS(_bd.blockSizeBytes); res["uncles"] = Json::Value(Json::arrayValue); for (h256 h: _us) res["uncles"].append(toJS(h)); From 5d792ef7f8a9b49b6e8a907447ed65f97b29359b Mon Sep 17 00:00:00 2001 From: Nils-Erik Frantzell Date: Thu, 21 Nov 2019 20:41:45 -0800 Subject: [PATCH 4/9] Detect and handle another extrasdb upgrade case Case = minor version file not present but extras database exists. This should only happen if someone is upgrading to a release of Aleth which writes the minor version file, and this release will also have an extras database upgrade so require a rebuild. --- libethereum/BlockChain.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index 18bedad6b66..72c9d42e870 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -230,6 +230,13 @@ bool BlockChain::open(fs::path const& _path, WithExisting _we) << ") != Aleth's version (" << c_databaseMinorVersion << ")"; } } + else if (fs::exists(extrasSubPathExtras)) + { + LOG(m_loggerInfo) << "Database minor version file not found (" << extrasSubPathMinor + << ") but extras database exists (" << extrasSubPathExtras + << "), assuming extras database needs to be upgraded."; + rebuildNeeded = true; + } else { LOG(m_loggerDetail) << "Creating database minor version file: " << extrasSubPathMinor From 15df0fcfe6b26238c4c1348d886af414f12225e0 Mon Sep 17 00:00:00 2001 From: Nils-Erik Frantzell Date: Mon, 25 Nov 2019 20:37:29 -0800 Subject: [PATCH 5/9] Move chain database paths into new namespace New namespace is called dev::eth::database_paths. Also tweak BlockChain::open and BlockChain::rebuild so that BlockChain::m_dbPath is no longer needed, and set BlockDetails::size in BlockDetails::rlp(). Finally, remove unnecessary boost::filesystem::create_directories calls in BlockChain::open / State::openDB. --- libethcore/Exceptions.h | 2 + libethereum/BlockChain.cpp | 86 +++++++++++++++++------------------ libethereum/BlockChain.h | 5 +- libethereum/BlockDetails.cpp | 5 +- libethereum/DatabasePaths.cpp | 85 ++++++++++++++++++++++++++++++++++ libethereum/DatabasePaths.h | 27 +++++++++++ libethereum/State.cpp | 34 ++++++-------- 7 files changed, 174 insertions(+), 70 deletions(-) create mode 100644 libethereum/DatabasePaths.cpp create mode 100644 libethereum/DatabasePaths.h diff --git a/libethcore/Exceptions.h b/libethcore/Exceptions.h index 6311f8ebbeb..042517ce30c 100644 --- a/libethcore/Exceptions.h +++ b/libethcore/Exceptions.h @@ -57,6 +57,7 @@ DEV_SIMPLE_EXCEPTION(InvalidLogBloom); DEV_SIMPLE_EXCEPTION(InvalidNonce); DEV_SIMPLE_EXCEPTION(InvalidBlockHeaderItemCount); DEV_SIMPLE_EXCEPTION(InvalidBlockNonce); +DEV_SIMPLE_EXCEPTION(InvalidHash); DEV_SIMPLE_EXCEPTION(InvalidParentHash); DEV_SIMPLE_EXCEPTION(InvalidUncleParentHash); DEV_SIMPLE_EXCEPTION(InvalidNumber); @@ -78,6 +79,7 @@ DEV_SIMPLE_EXCEPTION(DatabaseAlreadyOpen); DEV_SIMPLE_EXCEPTION(DatabaseCorruption); DEV_SIMPLE_EXCEPTION(DatabaseExists); DEV_SIMPLE_EXCEPTION(DatabaseRebuildFailed); +DEV_SIMPLE_EXCEPTION(DatabasePathsNotSet); DEV_SIMPLE_EXCEPTION(DAGCreationFailure); DEV_SIMPLE_EXCEPTION(DAGComputeFailure); diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index 72c9d42e870..0790ed4fc89 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -25,6 +26,7 @@ using namespace std; using namespace dev; using namespace dev::eth; namespace fs = boost::filesystem; +namespace db_paths = dev::eth::database_paths; #define ETH_TIMED_IMPORTS 1 @@ -153,8 +155,7 @@ static const unsigned c_minCacheSize = 1024 * 1024 * 32; BlockChain::BlockChain(ChainParams const& _p, fs::path const& _dbPath, WithExisting _we, ProgressCallback const& _pc): - m_lastBlockHashes(new LastBlockHashes(*this)), - m_dbPath(_dbPath) + m_lastBlockHashes(new LastBlockHashes(*this)) { init(_p); open(_dbPath, _we, _pc); @@ -194,30 +195,22 @@ void BlockChain::init(ChainParams const& _p) bool BlockChain::open(fs::path const& _path, WithExisting _we) { - auto const path = _path.empty() ? db::databasePath() : _path; - auto const chainPath = path / fs::path(toHex(m_genesisHash.ref().cropped(0, 4))); - auto const chainSubPathBlocks = chainPath / fs::path("blocks"); - auto const extrasPath = chainPath / fs::path(toString(c_databaseVersion)); - auto const extrasSubPathExtras = extrasPath / fs::path("extras"); unsigned lastMinor = c_databaseMinorVersion; bool rebuildNeeded = false; + db_paths::setDatabasePaths(_path, m_genesisHash); if (db::isDiskDatabase()) { if (_we == WithExisting::Kill) { LOG(m_loggerInfo) << "Deleting chain databases. This will require a resync from genesis."; - fs::remove_all(chainSubPathBlocks); - fs::remove_all(extrasSubPathExtras); - fs::remove_all(extrasPath / fs::path("extras.old")); + fs::remove_all(db_paths::blocksDatabasePath()); + fs::remove_all(db_paths::extrasDatabasePath()); + fs::remove_all(db_paths::extrasDatabaseTemporaryPath()); } - fs::create_directories(extrasPath); - DEV_IGNORE_EXCEPTIONS(fs::permissions(extrasPath, fs::owner_all)); - - auto const extrasSubPathMinor = extrasPath / fs::path("minor"); - bytes const minorVersionBytes = contents(extrasSubPathMinor); + bytes const minorVersionBytes = contents(db_paths::extrasDatabaseMinorVersionPath()); if (!minorVersionBytes.empty()) { DEV_IGNORE_EXCEPTIONS(lastMinor = (unsigned)RLP(minorVersionBytes)); @@ -226,40 +219,43 @@ bool BlockChain::open(fs::path const& _path, WithExisting _we) rebuildNeeded = true; LOG(m_loggerInfo) << "Database minor version change detected, the extras and state " "databases will be rebuilt."; - LOG(m_loggerInfo) << "Version from " << extrasSubPathMinor << " (" << lastMinor - << ") != Aleth's version (" << c_databaseMinorVersion << ")"; + LOG(m_loggerInfo) << "Version from " << db_paths::extrasDatabaseMinorVersionPath() + << " (" << lastMinor << ") != Aleth's version (" + << c_databaseMinorVersion << ")"; } } - else if (fs::exists(extrasSubPathExtras)) + else if (fs::exists(db_paths::extrasDatabasePath())) { - LOG(m_loggerInfo) << "Database minor version file not found (" << extrasSubPathMinor - << ") but extras database exists (" << extrasSubPathExtras + LOG(m_loggerInfo) << "Database minor version file not found (" + << db_paths::extrasDatabaseMinorVersionPath() + << ") but extras database exists (" << db_paths::extrasDatabasePath() << "), assuming extras database needs to be upgraded."; rebuildNeeded = true; } else { - LOG(m_loggerDetail) << "Creating database minor version file: " << extrasSubPathMinor + LOG(m_loggerDetail) << "Creating database minor version file: " + << db_paths::extrasDatabaseMinorVersionPath() << " (minor version: " << c_databaseMinorVersion << ")"; - writeFile(extrasSubPathMinor, rlp(c_databaseMinorVersion)); + writeFile(db_paths::extrasDatabaseMinorVersionPath(), rlp(c_databaseMinorVersion)); } } try { - m_blocksDB = db::DBFactory::create(chainSubPathBlocks); - m_extrasDB = db::DBFactory::create(extrasSubPathExtras); + m_blocksDB = db::DBFactory::create(db_paths::blocksDatabasePath()); + m_extrasDB = db::DBFactory::create(db_paths::extrasDatabasePath()); } catch (db::DatabaseError const& ex) { // Determine which database open call failed - auto const dbPath = !m_blocksDB.get() ? chainSubPathBlocks : extrasSubPathExtras; + auto const dbPath = !m_blocksDB.get() ? db_paths::blocksDatabasePath() : db_paths::extrasDatabasePath(); LOG(m_loggerError) << "Error opening database: " << dbPath; if (db::isDiskDatabase()) { db::DatabaseStatus const dbStatus = *boost::get_error_info(ex); - if (fs::space(path).available < 1024) + if (fs::space(db_paths::rootPath()).available < 1024) { LOG(m_loggerError) << "Not enough available space found on hard drive. Please free some up and " @@ -305,7 +301,8 @@ bool BlockChain::open(fs::path const& _path, WithExisting _we) // database because the extras database format may have changed m_lastBlockNumber = info(m_lastBlockHash).number(); - LOG(m_loggerInfo) << "Opened blockchain DB. Latest: " << currentHash() + LOG(m_loggerInfo) << "Opened blockchain database (" << db_paths::chainPath() + << "). Latest: " << currentHash() << (!rebuildNeeded ? "(rebuild not needed)" : "*** REBUILD NEEDED ***"); return rebuildNeeded; } @@ -320,7 +317,7 @@ void BlockChain::reopen(ChainParams const& _p, WithExisting _we, ProgressCallbac { close(); init(_p); - open(m_dbPath, _we, _pc); + open(db_paths::rootPath(), _we, _pc); } void BlockChain::close() @@ -346,7 +343,8 @@ void BlockChain::close() m_lastBlockHashes->clear(); } -void BlockChain::rebuild(fs::path const& _path, std::function const& _progress) +void BlockChain::rebuild( + fs::path const& _path, std::function const& _progress) { if (!db::isDiskDatabase()) { @@ -355,11 +353,7 @@ void BlockChain::rebuild(fs::path const& _path, std::function oldExtrasDB(db::DBFactory::create(extrasSubPathOldExtras)); - m_extrasDB = db::DBFactory::create(extrasSubPathExtras); + fs::rename(db_paths::extrasDatabasePath(), db_paths::extrasDatabaseTemporaryPath()); + std::unique_ptr oldExtrasDB(db::DBFactory::create(db_paths::extrasDatabaseTemporaryPath())); + m_extrasDB = db::DBFactory::create(db_paths::extrasDatabasePath()); // Open a fresh state DB - Block s = genesisBlock(State::openDB(path.string(), m_genesisHash, WithExisting::Kill)); + Block s = genesisBlock( + State::openDB(db_paths::rootPath().string(), m_genesisHash, WithExisting::Kill)); // Clear all memos ready for replay. m_details.clear(); @@ -456,13 +451,14 @@ void BlockChain::rebuild(fs::path const& _path, std::function()); + void rebuild(boost::filesystem::path const& _path, + ProgressCallback const& _progress = std::function()); /// Alter the head of the chain to some prior block along it. void rewind(unsigned _newHead); @@ -410,8 +411,6 @@ class BlockChain std::function m_onBad; ///< Called if we have a block that doesn't verify. std::function m_onBlockImport; ///< Called if we have imported a new block into the db - boost::filesystem::path m_dbPath; - mutable Logger m_logger{createLogger(VerbosityDebug, "chain")}; mutable Logger m_loggerDetail{createLogger(VerbosityTrace, "chain")}; mutable Logger m_loggerWarn{createLogger(VerbosityWarning, "chain")}; diff --git a/libethereum/BlockDetails.cpp b/libethereum/BlockDetails.cpp index 660f024abc8..ce13d27ed48 100644 --- a/libethereum/BlockDetails.cpp +++ b/libethereum/BlockDetails.cpp @@ -22,5 +22,8 @@ BlockDetails::BlockDetails(RLP const& _r) bytes BlockDetails::rlp() const { - return rlpList(number, totalDifficulty, parentHash, childHashes, blockSizeBytes); + auto const detailsRlp = + rlpList(number, totalDifficulty, parentHash, childHashes, blockSizeBytes); + size = detailsRlp.size(); + return detailsRlp; } diff --git a/libethereum/DatabasePaths.cpp b/libethereum/DatabasePaths.cpp new file mode 100644 index 00000000000..37a455645fd --- /dev/null +++ b/libethereum/DatabasePaths.cpp @@ -0,0 +1,85 @@ +// Aleth: Ethereum C++ client, tools and libraries. +// Copyright 2014-2019 Aleth Authors. +// Licensed under the GNU General Public License, Version 3. + +#include "DatabasePaths.h" +#include "libdevcore/CommonIO.h" +#include "libdevcore/DBFactory.h" +#include "libethcore/Exceptions.h" + +namespace dev +{ +namespace eth +{ +namespace database_paths +{ +namespace fs = boost::filesystem; + +fs::path g_rootPath; +fs::path g_chainPath; +fs::path g_blocksDbPath; +fs::path g_stateDbPath; +fs::path g_extrasDbPath; +fs::path g_extrasDbTempPath; +fs::path g_extrasDbMinorVersionPath; + +bool databasePathsSet() +{ + return !g_rootPath.empty(); +} + +void setDatabasePaths(fs::path const& _rootPath, h256 const& _genesisHash) +{ + if (_genesisHash == h256{}) + { + BOOST_THROW_EXCEPTION(InvalidHash()); + } + + g_rootPath = _rootPath.empty() ? db::databasePath() : _rootPath; + g_chainPath = g_rootPath / fs::path(toHex(_genesisHash.ref().cropped(0, 4))); + g_stateDbPath = g_chainPath / fs::path("state"); + g_blocksDbPath = g_chainPath / fs::path("blocks"); + + auto const extrasRootPath = g_chainPath / fs::path(toString(c_databaseVersion)); + g_extrasDbPath = extrasRootPath / fs::path("extras"); + g_extrasDbTempPath = extrasRootPath / fs::path("extras.old"); + g_extrasDbMinorVersionPath = g_extrasDbPath / fs::path("minor"); +} + +fs::path rootPath() +{ + return g_rootPath; +} + +fs::path chainPath() +{ + return g_chainPath; +} + +fs::path stateDatabasePath() +{ + return g_stateDbPath; +} + +fs::path blocksDatabasePath() +{ + return g_blocksDbPath; +} + +fs::path extrasDatabasePath() +{ + return g_extrasDbPath; +} + +fs::path extrasDatabaseTemporaryPath() +{ + return g_extrasDbTempPath; +} + +fs::path extrasDatabaseMinorVersionPath() +{ + return g_extrasDbMinorVersionPath; +} +} // namespace database_paths +} // namespace eth +} // namespace dev \ No newline at end of file diff --git a/libethereum/DatabasePaths.h b/libethereum/DatabasePaths.h new file mode 100644 index 00000000000..7034baada16 --- /dev/null +++ b/libethereum/DatabasePaths.h @@ -0,0 +1,27 @@ +// Aleth: Ethereum C++ client, tools and libraries. +// Copyright 2014-2019 Aleth Authors. +// Licensed under the GNU General Public License, Version 3. + +#pragma once + +#include "libdevcore/FixedHash.h" +#include + +namespace dev +{ +namespace eth +{ +namespace database_paths +{ +bool databasePathsSet(); +void setDatabasePaths(boost::filesystem::path const& _rootPath, h256 const& _genesisHash); +boost::filesystem::path rootPath(); +boost::filesystem::path chainPath(); +boost::filesystem::path blocksDatabasePath(); +boost::filesystem::path stateDatabasePath(); +boost::filesystem::path extrasDatabasePath(); +boost::filesystem::path extrasDatabaseTemporaryPath(); +boost::filesystem::path extrasDatabaseMinorVersionPath(); +} // namespace database_paths +} // namespace eth +} // namespace dev \ No newline at end of file diff --git a/libethereum/State.cpp b/libethereum/State.cpp index c6b753c68c8..c708d47ef1a 100644 --- a/libethereum/State.cpp +++ b/libethereum/State.cpp @@ -8,6 +8,7 @@ #include "BlockChain.h" #include "ExtVM.h" #include "TransactionQueue.h" +#include "DatabasePaths.h" #include #include #include @@ -18,6 +19,7 @@ using namespace std; using namespace dev; using namespace dev::eth; namespace fs = boost::filesystem; +namespace db_paths = database_paths; State::State(u256 const& _accountStartNonce, OverlayDB const& _db, BaseState _bs): m_db(_db), @@ -42,40 +44,30 @@ State::State(State const& _s): OverlayDB State::openDB(fs::path const& _basePath, h256 const& _genesisHash, WithExisting _we) { - auto const path = _basePath.empty() ? db::databasePath() : _basePath; - auto const chainPath = path / fs::path(toHex(_genesisHash.ref().cropped(0, 4))) / - fs::path(toString(c_databaseVersion)); - auto const statePath = chainPath / fs::path("state"); - - if (db::isDiskDatabase()) + db_paths::setDatabasePaths(_basePath); + if (db::isDiskDatabase() && _we == WithExisting::Kill) { - if (_we == WithExisting::Kill) - { - clog(VerbosityDebug, "statedb") - << "Killing state database (" << statePath << ") (WithExisting::Kill)"; - fs::remove_all(statePath); - } - - clog(VerbosityDebug, "statedb") << "Verifying path exists (and creating if not present): " << chainPath; - fs::create_directories(chainPath); - clog(VerbosityDebug, "statedb") << "Ensuring permissions are set for path: " << chainPath; - DEV_IGNORE_EXCEPTIONS(fs::permissions(chainPath, fs::owner_all)); + clog(VerbosityInfo, "statedb") + << "Deleting state database: " << db_paths::stateDatabasePath(); + fs::remove_all(db_paths::stateDatabasePath()); } try { - clog(VerbosityTrace, "statedb") << "Opening state database: " << statePath; - std::unique_ptr db = db::DBFactory::create(statePath); + clog(VerbosityTrace, "statedb") << "Opening state database (and creating if not present): " + << db_paths::stateDatabasePath(); + std::unique_ptr db = db::DBFactory::create(db_paths::stateDatabasePath()); return OverlayDB(std::move(db)); } catch (boost::exception const& ex) { - clog(VerbosityError, "statedb") << "Error opening state database: " << statePath; + clog(VerbosityError, "statedb") + << "Error opening state database: " << db_paths::stateDatabasePath(); if (db::isDiskDatabase()) { db::DatabaseStatus const dbStatus = *boost::get_error_info(ex); - if (fs::space(statePath).available < 1024) + if (fs::space(db_paths::stateDatabasePath()).available < 1024) { clog(VerbosityError, "statedb") << "Not enough available space found on hard drive. Please free some up and " From d2975f635dd06d26727c540a7c073b0bd6b4c4bb Mon Sep 17 00:00:00 2001 From: Nils-Erik Frantzell Date: Mon, 25 Nov 2019 22:01:16 -0800 Subject: [PATCH 6/9] Minor cleanup Remove misleading log messages when memoryDB is used Only set database paths if disk db is being used Remove unused exceptions --- libethcore/Exceptions.h | 2 -- libethereum/BlockChain.cpp | 21 +++++++++++---------- libethereum/DatabasePaths.cpp | 7 ++----- libethereum/State.cpp | 25 ++++++++++++++----------- 4 files changed, 27 insertions(+), 28 deletions(-) diff --git a/libethcore/Exceptions.h b/libethcore/Exceptions.h index 042517ce30c..6311f8ebbeb 100644 --- a/libethcore/Exceptions.h +++ b/libethcore/Exceptions.h @@ -57,7 +57,6 @@ DEV_SIMPLE_EXCEPTION(InvalidLogBloom); DEV_SIMPLE_EXCEPTION(InvalidNonce); DEV_SIMPLE_EXCEPTION(InvalidBlockHeaderItemCount); DEV_SIMPLE_EXCEPTION(InvalidBlockNonce); -DEV_SIMPLE_EXCEPTION(InvalidHash); DEV_SIMPLE_EXCEPTION(InvalidParentHash); DEV_SIMPLE_EXCEPTION(InvalidUncleParentHash); DEV_SIMPLE_EXCEPTION(InvalidNumber); @@ -79,7 +78,6 @@ DEV_SIMPLE_EXCEPTION(DatabaseAlreadyOpen); DEV_SIMPLE_EXCEPTION(DatabaseCorruption); DEV_SIMPLE_EXCEPTION(DatabaseExists); DEV_SIMPLE_EXCEPTION(DatabaseRebuildFailed); -DEV_SIMPLE_EXCEPTION(DatabasePathsNotSet); DEV_SIMPLE_EXCEPTION(DAGCreationFailure); DEV_SIMPLE_EXCEPTION(DAGComputeFailure); diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index 0790ed4fc89..fdf1198c702 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -198,9 +198,9 @@ bool BlockChain::open(fs::path const& _path, WithExisting _we) unsigned lastMinor = c_databaseMinorVersion; bool rebuildNeeded = false; - db_paths::setDatabasePaths(_path, m_genesisHash); if (db::isDiskDatabase()) { + db_paths::setDatabasePaths(_path, m_genesisHash); if (_we == WithExisting::Kill) { LOG(m_loggerInfo) @@ -234,6 +234,7 @@ bool BlockChain::open(fs::path const& _path, WithExisting _we) } else { + // First launch with new database LOG(m_loggerDetail) << "Creating database minor version file: " << db_paths::extrasDatabaseMinorVersionPath() << " (minor version: " << c_databaseMinorVersion << ")"; @@ -249,11 +250,11 @@ bool BlockChain::open(fs::path const& _path, WithExisting _we) catch (db::DatabaseError const& ex) { // Determine which database open call failed - auto const dbPath = !m_blocksDB.get() ? db_paths::blocksDatabasePath() : db_paths::extrasDatabasePath(); - LOG(m_loggerError) << "Error opening database: " << dbPath; - + auto const dbPath = + !m_blocksDB.get() ? db_paths::blocksDatabasePath() : db_paths::extrasDatabasePath(); if (db::isDiskDatabase()) { + LOG(m_loggerError) << "Error occurred when opening database: " << dbPath; db::DatabaseStatus const dbStatus = *boost::get_error_info(ex); if (fs::space(db_paths::rootPath()).available < 1024) { @@ -278,7 +279,8 @@ bool BlockChain::open(fs::path const& _path, WithExisting _we) } } - LOG(m_loggerError) << "Unknown error occurred. Exception details: " << ex.what(); + LOG(m_loggerError) << "Unknown error occurred when opening database. Exception details: " + << ex.what(); throw; } @@ -301,8 +303,7 @@ bool BlockChain::open(fs::path const& _path, WithExisting _we) // database because the extras database format may have changed m_lastBlockNumber = info(m_lastBlockHash).number(); - LOG(m_loggerInfo) << "Opened blockchain database (" << db_paths::chainPath() - << "). Latest: " << currentHash() + LOG(m_loggerInfo) << "Opened blockchain database. Latest block hash: " << currentHash() << (!rebuildNeeded ? "(rebuild not needed)" : "*** REBUILD NEEDED ***"); return rebuildNeeded; } @@ -377,12 +378,12 @@ void BlockChain::rebuild( BOOST_THROW_EXCEPTION(DatabaseExists()); } fs::rename(db_paths::extrasDatabasePath(), db_paths::extrasDatabaseTemporaryPath()); - std::unique_ptr oldExtrasDB(db::DBFactory::create(db_paths::extrasDatabaseTemporaryPath())); + std::unique_ptr oldExtrasDB( + db::DBFactory::create(db_paths::extrasDatabaseTemporaryPath())); m_extrasDB = db::DBFactory::create(db_paths::extrasDatabasePath()); // Open a fresh state DB - Block s = genesisBlock( - State::openDB(db_paths::rootPath().string(), m_genesisHash, WithExisting::Kill)); + Block s = genesisBlock(State::openDB(db_paths::rootPath(), m_genesisHash, WithExisting::Kill)); // Clear all memos ready for replay. m_details.clear(); diff --git a/libethereum/DatabasePaths.cpp b/libethereum/DatabasePaths.cpp index 37a455645fd..a3d3ad15e00 100644 --- a/libethereum/DatabasePaths.cpp +++ b/libethereum/DatabasePaths.cpp @@ -30,11 +30,8 @@ bool databasePathsSet() void setDatabasePaths(fs::path const& _rootPath, h256 const& _genesisHash) { - if (_genesisHash == h256{}) - { - BOOST_THROW_EXCEPTION(InvalidHash()); - } - + // Allow empty hashes since they are required by tests + g_rootPath = _rootPath.empty() ? db::databasePath() : _rootPath; g_chainPath = g_rootPath / fs::path(toHex(_genesisHash.ref().cropped(0, 4))); g_stateDbPath = g_chainPath / fs::path("state"); diff --git a/libethereum/State.cpp b/libethereum/State.cpp index c708d47ef1a..325af2841c9 100644 --- a/libethereum/State.cpp +++ b/libethereum/State.cpp @@ -44,27 +44,29 @@ State::State(State const& _s): OverlayDB State::openDB(fs::path const& _basePath, h256 const& _genesisHash, WithExisting _we) { - db_paths::setDatabasePaths(_basePath); - if (db::isDiskDatabase() && _we == WithExisting::Kill) + if (db::isDiskDatabase()) { - clog(VerbosityInfo, "statedb") - << "Deleting state database: " << db_paths::stateDatabasePath(); - fs::remove_all(db_paths::stateDatabasePath()); + db_paths::setDatabasePaths(_basePath, _genesisHash); + if (_we == WithExisting::Kill) + { + clog(VerbosityInfo, "statedb") + << "Deleting state database: " << db_paths::stateDatabasePath(); + fs::remove_all(db_paths::stateDatabasePath()); + } } try { - clog(VerbosityTrace, "statedb") << "Opening state database (and creating if not present): " - << db_paths::stateDatabasePath(); + clog(VerbosityTrace, "statedb") << "Opening state database"; std::unique_ptr db = db::DBFactory::create(db_paths::stateDatabasePath()); return OverlayDB(std::move(db)); } catch (boost::exception const& ex) { - clog(VerbosityError, "statedb") - << "Error opening state database: " << db_paths::stateDatabasePath(); if (db::isDiskDatabase()) { + clog(VerbosityError, "statedb") + << "Error opening state database: " << db_paths::stateDatabasePath(); db::DatabaseStatus const dbStatus = *boost::get_error_info(ex); if (fs::space(db_paths::stateDatabasePath()).available < 1024) @@ -89,8 +91,9 @@ OverlayDB State::openDB(fs::path const& _basePath, h256 const& _genesisHash, Wit BOOST_THROW_EXCEPTION(DatabaseAlreadyOpen()); } } - clog(VerbosityError, "statedb") << "Unknown error encountered. Exception details: " - << boost::diagnostic_information(ex); + clog(VerbosityError, "statedb") + << "Unknown error encountered when opening state database. Exception details: " + << boost::diagnostic_information(ex); throw; } } From 075c18ff967308cec9e7059c51978dfe5417d9e6 Mon Sep 17 00:00:00 2001 From: Nils-Erik Frantzell Date: Tue, 26 Nov 2019 21:00:37 -0800 Subject: [PATCH 7/9] Explicitly ensure that state db path is present Required by one of the tests (BlockChainFrontierSuite/opendb) --- libethereum/State.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/libethereum/State.cpp b/libethereum/State.cpp index 325af2841c9..b32dd054c56 100644 --- a/libethereum/State.cpp +++ b/libethereum/State.cpp @@ -53,6 +53,13 @@ OverlayDB State::openDB(fs::path const& _basePath, h256 const& _genesisHash, Wit << "Deleting state database: " << db_paths::stateDatabasePath(); fs::remove_all(db_paths::stateDatabasePath()); } + + clog(VerbosityDebug, "statedb") + << "Verifying path exists (and creating if not present): " << db_paths::chainPath(); + fs::create_directories(db_paths::chainPath()); + clog(VerbosityDebug, "statedb") + << "Ensuring permissions are set for path: " << db_paths::chainPath(); + DEV_IGNORE_EXCEPTIONS(fs::permissions(db_paths::chainPath(), fs::owner_all)); } try From a19f2f80d2dc7252a9438a1fbff1afa292759c20 Mon Sep 17 00:00:00 2001 From: Nils-Erik Frantzell Date: Wed, 27 Nov 2019 21:32:59 -0800 Subject: [PATCH 8/9] Address PR feedback Remove unnecessary static_cast when creating BlockDetails and change return values of DatabasePaths getters to const& Use const and noexcept where possible and move DatabasePaths from namespace to immutable class --- libethereum/BlockChain.cpp | 74 ++++++++++++++++++----------------- libethereum/BlockChain.h | 9 +++-- libethereum/ClientBase.cpp | 8 ++-- libethereum/DatabasePaths.cpp | 68 ++++++++++++-------------------- libethereum/DatabasePaths.h | 31 +++++++++------ libethereum/State.cpp | 22 +++++------ 6 files changed, 103 insertions(+), 109 deletions(-) diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index fdf1198c702..02c0d05d34e 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -26,7 +26,6 @@ using namespace std; using namespace dev; using namespace dev::eth; namespace fs = boost::filesystem; -namespace db_paths = dev::eth::database_paths; #define ETH_TIMED_IMPORTS 1 @@ -200,17 +199,19 @@ bool BlockChain::open(fs::path const& _path, WithExisting _we) if (db::isDiskDatabase()) { - db_paths::setDatabasePaths(_path, m_genesisHash); + if (!m_dbPaths || m_dbPaths->rootPath() != _path) + m_dbPaths = make_unique(_path, m_genesisHash); + if (_we == WithExisting::Kill) { LOG(m_loggerInfo) << "Deleting chain databases. This will require a resync from genesis."; - fs::remove_all(db_paths::blocksDatabasePath()); - fs::remove_all(db_paths::extrasDatabasePath()); - fs::remove_all(db_paths::extrasDatabaseTemporaryPath()); + fs::remove_all(m_dbPaths->blocksPath()); + fs::remove_all(m_dbPaths->extrasPath()); + fs::remove_all(m_dbPaths->extrasTemporaryPath()); } - bytes const minorVersionBytes = contents(db_paths::extrasDatabaseMinorVersionPath()); + bytes const minorVersionBytes = contents(m_dbPaths->extrasMinorVersionPath()); if (!minorVersionBytes.empty()) { DEV_IGNORE_EXCEPTIONS(lastMinor = (unsigned)RLP(minorVersionBytes)); @@ -219,16 +220,16 @@ bool BlockChain::open(fs::path const& _path, WithExisting _we) rebuildNeeded = true; LOG(m_loggerInfo) << "Database minor version change detected, the extras and state " "databases will be rebuilt."; - LOG(m_loggerInfo) << "Version from " << db_paths::extrasDatabaseMinorVersionPath() + LOG(m_loggerInfo) << "Version from " << m_dbPaths->extrasMinorVersionPath() << " (" << lastMinor << ") != Aleth's version (" << c_databaseMinorVersion << ")"; } } - else if (fs::exists(db_paths::extrasDatabasePath())) + else if (fs::exists(m_dbPaths->extrasPath())) { LOG(m_loggerInfo) << "Database minor version file not found (" - << db_paths::extrasDatabaseMinorVersionPath() - << ") but extras database exists (" << db_paths::extrasDatabasePath() + << m_dbPaths->extrasMinorVersionPath() + << ") but extras database exists (" << m_dbPaths->extrasPath() << "), assuming extras database needs to be upgraded."; rebuildNeeded = true; } @@ -236,27 +237,27 @@ bool BlockChain::open(fs::path const& _path, WithExisting _we) { // First launch with new database LOG(m_loggerDetail) << "Creating database minor version file: " - << db_paths::extrasDatabaseMinorVersionPath() + << m_dbPaths->extrasMinorVersionPath() << " (minor version: " << c_databaseMinorVersion << ")"; - writeFile(db_paths::extrasDatabaseMinorVersionPath(), rlp(c_databaseMinorVersion)); + writeFile(m_dbPaths->extrasMinorVersionPath(), rlp(c_databaseMinorVersion)); } } try { - m_blocksDB = db::DBFactory::create(db_paths::blocksDatabasePath()); - m_extrasDB = db::DBFactory::create(db_paths::extrasDatabasePath()); + m_blocksDB = db::DBFactory::create(m_dbPaths->blocksPath()); + m_extrasDB = db::DBFactory::create(m_dbPaths->extrasPath()); } catch (db::DatabaseError const& ex) { // Determine which database open call failed auto const dbPath = - !m_blocksDB.get() ? db_paths::blocksDatabasePath() : db_paths::extrasDatabasePath(); + !m_blocksDB.get() ? m_dbPaths->blocksPath() : m_dbPaths->extrasPath(); if (db::isDiskDatabase()) { LOG(m_loggerError) << "Error occurred when opening database: " << dbPath; db::DatabaseStatus const dbStatus = *boost::get_error_info(ex); - if (fs::space(db_paths::rootPath()).available < 1024) + if (fs::space(m_dbPaths->rootPath()).available < 1024) { LOG(m_loggerError) << "Not enough available space found on hard drive. Please free some up and " @@ -287,13 +288,14 @@ bool BlockChain::open(fs::path const& _path, WithExisting _we) if (_we != WithExisting::Verify && !rebuildNeeded && !details(m_genesisHash)) { bytes const genesisBlockBytes = m_params.genesisBlock(); - BlockHeader gb{genesisBlockBytes}; + BlockHeader const genesisHeader{genesisBlockBytes}; // Insert details of genesis block. - m_details[m_genesisHash] = BlockDetails{0 /* number */, gb.difficulty(), + m_details[m_genesisHash] = BlockDetails{0 /* number */, genesisHeader.difficulty(), h256{} /* parent */, {} /* children */, genesisBlockBytes.size()}; - auto r = m_details[m_genesisHash].rlp(); - m_extrasDB->insert(toSlice(m_genesisHash, ExtraDetails), (db::Slice)dev::ref(r)); - assert(isKnown(gb.hash())); + auto const genesisDetailsRlp = m_details[m_genesisHash].rlp(); + m_extrasDB->insert( + toSlice(m_genesisHash, ExtraDetails), (db::Slice)dev::ref(genesisDetailsRlp)); + assert(isKnown(genesisHeader.hash())); } // TODO: Implement ability to rebuild details map from DB. @@ -318,7 +320,7 @@ void BlockChain::reopen(ChainParams const& _p, WithExisting _we, ProgressCallbac { close(); init(_p); - open(db_paths::rootPath(), _we, _pc); + open(m_dbPaths->rootPath(), _we, _pc); } void BlockChain::close() @@ -354,7 +356,8 @@ void BlockChain::rebuild( return; } - db_paths::setDatabasePaths(_path, m_genesisHash); + if (!m_dbPaths || m_dbPaths->rootPath() != _path) + m_dbPaths = make_unique(_path, m_genesisHash); unsigned const originalNumber = m_lastBlockNumber; @@ -365,25 +368,25 @@ void BlockChain::rebuild( // Keep extras DB around, but under a temp name m_extrasDB.reset(); - LOG(m_loggerInfo) << "Renaming extras path " << db_paths::extrasDatabasePath() << " to " - << db_paths::extrasDatabaseTemporaryPath(); - if (fs::exists(db_paths::extrasDatabaseTemporaryPath())) + LOG(m_loggerInfo) << "Renaming extras path " << m_dbPaths->extrasPath() << " to " + << m_dbPaths->extrasTemporaryPath(); + if (fs::exists(m_dbPaths->extrasTemporaryPath())) { LOG(m_loggerError) - << "Temporary extras path " << db_paths::extrasDatabaseTemporaryPath() + << "Temporary extras path " << m_dbPaths->extrasTemporaryPath() << " already exists (this usually happens because an in-progress rebuild was " "prematurely terminated)."; LOG(m_loggerError) << "Please re-run Aleth the --kill option to delete all databases. This " "will remove all chain data and require you to resync from genesis."; BOOST_THROW_EXCEPTION(DatabaseExists()); } - fs::rename(db_paths::extrasDatabasePath(), db_paths::extrasDatabaseTemporaryPath()); - std::unique_ptr oldExtrasDB( - db::DBFactory::create(db_paths::extrasDatabaseTemporaryPath())); - m_extrasDB = db::DBFactory::create(db_paths::extrasDatabasePath()); + fs::rename(m_dbPaths->extrasPath(), m_dbPaths->extrasTemporaryPath()); + std::unique_ptr oldExtrasDB{ + db::DBFactory::create(m_dbPaths->extrasTemporaryPath())}; + m_extrasDB = db::DBFactory::create(m_dbPaths->extrasPath()); // Open a fresh state DB - Block s = genesisBlock(State::openDB(db_paths::rootPath(), m_genesisHash, WithExisting::Kill)); + Block s = genesisBlock(State::openDB(m_dbPaths->rootPath(), m_genesisHash, WithExisting::Kill)); // Clear all memos ready for replay. m_details.clear(); @@ -452,14 +455,13 @@ void BlockChain::rebuild( if (_progress) _progress(d, originalNumber); } - LOG(m_loggerInfo) << "Removing old extras database: " - << db_paths::extrasDatabaseTemporaryPath(); + LOG(m_loggerInfo) << "Removing old extras database: " << m_dbPaths->extrasTemporaryPath(); oldExtrasDB.reset(); - fs::remove_all(db_paths::extrasDatabaseTemporaryPath()); + fs::remove_all(m_dbPaths->extrasTemporaryPath()); if (!rebuildFailed) { LOG(m_loggerInfo) << "Rebuild complete! Reimported " << originalNumber << " blocks!"; - writeFile(db_paths::extrasDatabaseMinorVersionPath(), rlp(c_databaseMinorVersion)); + writeFile(m_dbPaths->extrasMinorVersionPath(), rlp(c_databaseMinorVersion)); } else { diff --git a/libethereum/BlockChain.h b/libethereum/BlockChain.h index 2445b196dcd..cd60f4e9ab4 100644 --- a/libethereum/BlockChain.h +++ b/libethereum/BlockChain.h @@ -8,22 +8,23 @@ #include "BlockDetails.h" #include "BlockQueue.h" #include "ChainParams.h" +#include "DatabasePaths.h" #include "LastBlockHashesFace.h" #include "State.h" #include "Transaction.h" #include "VerifiedBlock.h" -#include #include -#include #include +#include +#include #include #include #include +#include #include #include #include #include -#include namespace std { @@ -408,6 +409,8 @@ class BlockChain mutable bytes m_genesisHeaderBytes; // mutable because they're effectively memos. mutable h256 m_genesisHash; // mutable because they're effectively memos. + std::unique_ptr m_dbPaths; // Paths for various databases (e.g. blocks, extras) + std::function m_onBad; ///< Called if we have a block that doesn't verify. std::function m_onBlockImport; ///< Called if we have imported a new block into the db diff --git a/libethereum/ClientBase.cpp b/libethereum/ClientBase.cpp index 48b6761df1d..1a3c5af06b2 100644 --- a/libethereum/ClientBase.cpp +++ b/libethereum/ClientBase.cpp @@ -417,11 +417,11 @@ BlockHeader ClientBase::pendingInfo() const BlockDetails ClientBase::pendingDetails() const { - auto pendingHeader = postSeal().info(); - auto latestDetails = Interface::blockDetails(LatestBlock); - return BlockDetails{(unsigned)pendingHeader.number(), + auto const pendingHeader = postSeal().info(); + auto const latestDetails = Interface::blockDetails(LatestBlock); + return BlockDetails{static_cast(pendingHeader.number()), latestDetails.totalDifficulty + pendingHeader.difficulty(), pendingHeader.parentHash(), - h256s{} /* children */, static_cast(postSeal().blockData().size())}; + h256s{} /* children */, postSeal().blockData().size()}; } Addresses ClientBase::addresses(BlockNumber _block) const diff --git a/libethereum/DatabasePaths.cpp b/libethereum/DatabasePaths.cpp index a3d3ad15e00..0f7c2944fab 100644 --- a/libethereum/DatabasePaths.cpp +++ b/libethereum/DatabasePaths.cpp @@ -4,79 +4,61 @@ #include "DatabasePaths.h" #include "libdevcore/CommonIO.h" -#include "libdevcore/DBFactory.h" -#include "libethcore/Exceptions.h" +#include "libethcore/Common.h" namespace dev { namespace eth { -namespace database_paths -{ namespace fs = boost::filesystem; -fs::path g_rootPath; -fs::path g_chainPath; -fs::path g_blocksDbPath; -fs::path g_stateDbPath; -fs::path g_extrasDbPath; -fs::path g_extrasDbTempPath; -fs::path g_extrasDbMinorVersionPath; - -bool databasePathsSet() -{ - return !g_rootPath.empty(); -} - -void setDatabasePaths(fs::path const& _rootPath, h256 const& _genesisHash) +DatabasePaths::DatabasePaths(fs::path const& _rootPath, h256 const& _genesisHash) noexcept { - // Allow empty hashes since they are required by tests - - g_rootPath = _rootPath.empty() ? db::databasePath() : _rootPath; - g_chainPath = g_rootPath / fs::path(toHex(_genesisHash.ref().cropped(0, 4))); - g_stateDbPath = g_chainPath / fs::path("state"); - g_blocksDbPath = g_chainPath / fs::path("blocks"); + // Allow an empty root path and empty genesis hash since they are required by the tests + m_rootPath = _rootPath; + m_chainPath = m_rootPath / fs::path(toHex(_genesisHash.ref().cropped(0, 4))); + m_statePath = m_chainPath / fs::path("state"); + m_blocksPath = m_chainPath / fs::path("blocks"); - auto const extrasRootPath = g_chainPath / fs::path(toString(c_databaseVersion)); - g_extrasDbPath = extrasRootPath / fs::path("extras"); - g_extrasDbTempPath = extrasRootPath / fs::path("extras.old"); - g_extrasDbMinorVersionPath = g_extrasDbPath / fs::path("minor"); + auto const extrasRootPath = m_chainPath / fs::path(toString(c_databaseVersion)); + m_extrasPath = extrasRootPath / fs::path("extras"); + m_extrasTemporaryPath = extrasRootPath / fs::path("extras.old"); + m_extrasMinorVersionPath = m_extrasPath / fs::path("minor"); } -fs::path rootPath() +fs::path const& DatabasePaths::rootPath() const noexcept { - return g_rootPath; + return m_rootPath; } -fs::path chainPath() +fs::path const& DatabasePaths::chainPath() const noexcept { - return g_chainPath; + return m_chainPath; } -fs::path stateDatabasePath() +fs::path const& DatabasePaths::statePath() const noexcept { - return g_stateDbPath; + return m_statePath; } -fs::path blocksDatabasePath() +fs::path const& DatabasePaths::blocksPath() const noexcept { - return g_blocksDbPath; + return m_blocksPath; } -fs::path extrasDatabasePath() +fs::path const& DatabasePaths::extrasPath() const noexcept { - return g_extrasDbPath; + return m_extrasPath; } -fs::path extrasDatabaseTemporaryPath() +fs::path const& DatabasePaths::extrasTemporaryPath() const noexcept { - return g_extrasDbTempPath; + return m_extrasTemporaryPath; } -fs::path extrasDatabaseMinorVersionPath() +fs::path const& DatabasePaths::extrasMinorVersionPath() const noexcept { - return g_extrasDbMinorVersionPath; + return m_extrasMinorVersionPath; } -} // namespace database_paths } // namespace eth } // namespace dev \ No newline at end of file diff --git a/libethereum/DatabasePaths.h b/libethereum/DatabasePaths.h index 7034baada16..cbd35d5f1c3 100644 --- a/libethereum/DatabasePaths.h +++ b/libethereum/DatabasePaths.h @@ -11,17 +11,26 @@ namespace dev { namespace eth { -namespace database_paths +class DatabasePaths { -bool databasePathsSet(); -void setDatabasePaths(boost::filesystem::path const& _rootPath, h256 const& _genesisHash); -boost::filesystem::path rootPath(); -boost::filesystem::path chainPath(); -boost::filesystem::path blocksDatabasePath(); -boost::filesystem::path stateDatabasePath(); -boost::filesystem::path extrasDatabasePath(); -boost::filesystem::path extrasDatabaseTemporaryPath(); -boost::filesystem::path extrasDatabaseMinorVersionPath(); -} // namespace database_paths +public: + DatabasePaths(boost::filesystem::path const& _rootPath, h256 const& _genesisHash) noexcept; + boost::filesystem::path const& rootPath() const noexcept; + boost::filesystem::path const& chainPath() const noexcept; + boost::filesystem::path const& blocksPath() const noexcept; + boost::filesystem::path const& statePath() const noexcept; + boost::filesystem::path const& extrasPath() const noexcept; + boost::filesystem::path const& extrasTemporaryPath() const noexcept; + boost::filesystem::path const& extrasMinorVersionPath() const noexcept; + +private: + boost::filesystem::path m_rootPath; + boost::filesystem::path m_chainPath; + boost::filesystem::path m_blocksPath; + boost::filesystem::path m_statePath; + boost::filesystem::path m_extrasPath; + boost::filesystem::path m_extrasTemporaryPath; + boost::filesystem::path m_extrasMinorVersionPath; +}; } // namespace eth } // namespace dev \ No newline at end of file diff --git a/libethereum/State.cpp b/libethereum/State.cpp index b32dd054c56..cad758f44a8 100644 --- a/libethereum/State.cpp +++ b/libethereum/State.cpp @@ -19,7 +19,6 @@ using namespace std; using namespace dev; using namespace dev::eth; namespace fs = boost::filesystem; -namespace db_paths = database_paths; State::State(u256 const& _accountStartNonce, OverlayDB const& _db, BaseState _bs): m_db(_db), @@ -44,28 +43,27 @@ State::State(State const& _s): OverlayDB State::openDB(fs::path const& _basePath, h256 const& _genesisHash, WithExisting _we) { + DatabasePaths const dbPaths{_basePath, _genesisHash}; if (db::isDiskDatabase()) { - db_paths::setDatabasePaths(_basePath, _genesisHash); if (_we == WithExisting::Kill) { - clog(VerbosityInfo, "statedb") - << "Deleting state database: " << db_paths::stateDatabasePath(); - fs::remove_all(db_paths::stateDatabasePath()); + clog(VerbosityInfo, "statedb") << "Deleting state database: " << dbPaths.statePath(); + fs::remove_all(dbPaths.statePath()); } clog(VerbosityDebug, "statedb") - << "Verifying path exists (and creating if not present): " << db_paths::chainPath(); - fs::create_directories(db_paths::chainPath()); + << "Verifying path exists (and creating if not present): " << dbPaths.chainPath(); + fs::create_directories(dbPaths.chainPath()); clog(VerbosityDebug, "statedb") - << "Ensuring permissions are set for path: " << db_paths::chainPath(); - DEV_IGNORE_EXCEPTIONS(fs::permissions(db_paths::chainPath(), fs::owner_all)); + << "Ensuring permissions are set for path: " << dbPaths.chainPath(); + DEV_IGNORE_EXCEPTIONS(fs::permissions(dbPaths.chainPath(), fs::owner_all)); } try { clog(VerbosityTrace, "statedb") << "Opening state database"; - std::unique_ptr db = db::DBFactory::create(db_paths::stateDatabasePath()); + std::unique_ptr db = db::DBFactory::create(dbPaths.statePath()); return OverlayDB(std::move(db)); } catch (boost::exception const& ex) @@ -73,10 +71,10 @@ OverlayDB State::openDB(fs::path const& _basePath, h256 const& _genesisHash, Wit if (db::isDiskDatabase()) { clog(VerbosityError, "statedb") - << "Error opening state database: " << db_paths::stateDatabasePath(); + << "Error opening state database: " << dbPaths.statePath(); db::DatabaseStatus const dbStatus = *boost::get_error_info(ex); - if (fs::space(db_paths::stateDatabasePath()).available < 1024) + if (fs::space(dbPaths.statePath()).available < 1024) { clog(VerbosityError, "statedb") << "Not enough available space found on hard drive. Please free some up and " From cfe3ce7fd58e3601de04f85a70530cf75f38d03b Mon Sep 17 00:00:00 2001 From: Nils-Erik Frantzell Date: Sun, 17 Nov 2019 21:57:18 -0800 Subject: [PATCH 9/9] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35c11d2b824..481e90dc8ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - Fixed: [#5826](https://github.com/ethereum/aleth/pull/5826) Fix blocking bug in database rebuild functionality - users can now rebuild their databases via Aleth's '-R' switch. - Fixed: [#5827](https://github.com/ethereum/aleth/pull/5827) Detect database upgrades and automatically rebuild the database when they occur. - Fixed: [#5852](https://github.com/ethereum/aleth/pull/5852) Output correct original opcodes instead of synthetic `PUSHC`/`JUMPC`/`JUMPCI` in VM trace. +- Fixed: [#5829](https://github.com/ethereum/aleth/pull/5829) web3.eth.getBlock now returns block size in bytes. This requires a (automatic) database rebuild which can take a while depending on how many blocks are in the local chain. ## [1.7.2] - 2019-11-22