Consensus Engines Implementation: Aura (#911)

* Generalize BlockImport

 - move ImportBlock, BlockOrigin, ImportResult into shared sr-primitives
 - let Consensus provide  and  traits again
 - update consensus traits to latest development
 - implement traits on client::Client, test_client::TestClient
 - update RHD to use the new import_block API

* Move ImportBlock into consensus-common
* Send import notification in aura tests
* Integrating aura into service
* Make Signatures more generic
* Aura Block Production with the given key
* run aura on the thread pool
* start at exact step start in aura
* Add needed wasm blob, in leiu of better solutions.
* Make API ids consistent with traits and bring upstream for sharing.
* Add decrease_free_balance to Balances module
* Encode `Metadata` once instead of two times
* Bitops include xor
* Upgrade key module.
* Default pages to somewhat bigger.
* Introduce upgrade key into node
* Add `Created` event

Cargo.toml

@@ -23,6 +23,7 @@ members = [
 	"core/client",
 	"core/client/db",
 	"core/consensus/common",
+	"core/consensus/aura",
 	"core/consensus/rhd",
 	"core/executor",
 	"core/finality-grandpa",
@@ -54,6 +55,7 @@ members = [
 	"srml/system",
 	"srml/timestamp",
 	"srml/treasury",
+	"srml/upgrade-key",
 	"core/serializer",
 	"core/service",
 	"core/service/test",
@@ -64,9 +66,7 @@ members = [
 	"core/trie",
 	"core/keystore",
 	"node/cli",
-	"node/consensus",
 	"node/executor",
-	"node/network",
 	"node/primitives",
 	"node/runtime",
 	"subkey",

core/client/Cargo.toml

@@ -12,7 +12,7 @@ hex-literal = "0.1"
 futures = "0.1.17"
 slog = "^2"
 heapsize = "0.4"
-substrate-consensus-rhd = { path = "../consensus/rhd" }
+substrate-consensus-common = { path = "../consensus/common" }
 parity-codec = "2.1"
 substrate-executor = { path = "../executor" }
 substrate-primitives = { path = "../primitives" }

core/client/db/src/lib.rs

@@ -534,7 +534,9 @@ impl<Block: BlockT> Backend<Block> {
 						meta.finalized_hash, f_hash),
 				).into())
 			}
-			transaction.put(columns::META, meta_keys::FINALIZED_BLOCK, f_hash.as_ref());
+
+			let lookup_key = ::utils::number_to_lookup_key(f_num);
+			transaction.put(columns::META, meta_keys::FINALIZED_BLOCK, &lookup_key);
 
 			let commit = self.storage.state_db.canonicalize_block(&f_hash);
 			apply_state_commit(transaction, commit);
@@ -586,11 +588,20 @@ impl<Block> client::backend::Backend<Block, Blake2Hasher> for Backend<Block> whe
 		-> Result<(), client::error::Error>
 	{
 		let mut transaction = DBTransaction::new();
+
 		if let Some(pending_block) = operation.pending_block {
 			let hash = pending_block.header.hash();
 			let parent_hash = *pending_block.header.parent_hash();
 			let number = pending_block.header.number().clone();
 
+			// blocks in longest chain are keyed by number
+			let lookup_key = if pending_block.leaf_state.is_best() {
+				::utils::number_to_lookup_key(number).to_vec()
+			} else {
+				// other blocks are keyed by number + hash
+				::utils::number_and_hash_to_lookup_key(number, hash)
+			};
+
 			if pending_block.leaf_state.is_best() {
 				let meta = self.blockchain.meta.read();
 
@@ -678,17 +689,9 @@ impl<Block> client::backend::Backend<Block, Blake2Hasher> for Backend<Block> whe
 					}
 				}
 
-				transaction.put(columns::META, meta_keys::BEST_BLOCK, hash.as_ref());
+				transaction.put(columns::META, meta_keys::BEST_BLOCK, &lookup_key);
 			}
 
-			// blocks in longest chain are keyed by number
-			let lookup_key = if pending_block.leaf_state.is_best() {
-				::utils::number_to_lookup_key(number).to_vec()
-			} else {
-			// other blocks are keyed by number + hash
-				::utils::number_and_hash_to_lookup_key(number, hash)
-			};
-
 			transaction.put(columns::HEADER, &lookup_key, &pending_block.header.encode());
 			if let Some(body) = pending_block.body {
 				transaction.put(columns::BODY, &lookup_key, &body.encode());
@@ -700,7 +703,7 @@ impl<Block> client::backend::Backend<Block, Blake2Hasher> for Backend<Block> whe
 			transaction.put(columns::HASH_LOOKUP, hash.as_ref(), &lookup_key);
 
 			if number == Zero::zero() {
-				transaction.put(columns::META, meta_keys::FINALIZED_BLOCK, hash.as_ref());
+				transaction.put(columns::META, meta_keys::FINALIZED_BLOCK, &lookup_key);
 				transaction.put(columns::META, meta_keys::GENESIS_HASH, hash.as_ref());
 			}
 
@@ -797,7 +800,8 @@ impl<Block> client::backend::Backend<Block, Blake2Hasher> for Backend<Block> whe
 						|| client::error::ErrorKind::UnknownBlock(
 							format!("Error reverting to {}. Block header not found.", best)))?;
 
-					transaction.put(columns::META, meta_keys::BEST_BLOCK, header.hash().as_ref());
+					let lookup_key = ::utils::number_to_lookup_key(header.number().clone());
+					transaction.put(columns::META, meta_keys::BEST_BLOCK, &lookup_key);
 					transaction.delete(columns::HASH_LOOKUP, header.hash().as_ref());
 					self.storage.db.write(transaction).map_err(db_err)?;
 					self.blockchain.update_meta(header.hash().clone(), best.clone(), true, false);
@@ -927,40 +931,49 @@ mod tests {
 
 	#[test]
 	fn block_hash_inserted_correctly() {
-		let db = Backend::<Block>::new_test(1, 0);
-		for i in 0..10 {
-			assert!(db.blockchain().hash(i).unwrap().is_none());
-
-			{
-				let id = if i == 0 {
-					BlockId::Hash(Default::default())
-				} else {
-					BlockId::Number(i - 1)
-				};
-
-				let mut op = db.begin_operation(id).unwrap();
-				let header = Header {
-					number: i,
-					parent_hash: if i == 0 {
-						Default::default()
+		let backing = {
+			let db = Backend::<Block>::new_test(1, 0);
+			for i in 0..10 {
+				assert!(db.blockchain().hash(i).unwrap().is_none());
+
+				{
+					let id = if i == 0 {
+						BlockId::Hash(Default::default())
 					} else {
-						db.blockchain.hash(i - 1).unwrap().unwrap()
-					},
-					state_root: Default::default(),
-					digest: Default::default(),
-					extrinsics_root: Default::default(),
-				};
-
-				op.set_block_data(
-					header,
-					Some(vec![]),
-					None,
-					NewBlockState::Best,
-				).unwrap();
-				db.commit_operation(op).unwrap();
+						BlockId::Number(i - 1)
+					};
+
+					let mut op = db.begin_operation(id).unwrap();
+					let header = Header {
+						number: i,
+						parent_hash: if i == 0 {
+							Default::default()
+						} else {
+							db.blockchain.hash(i - 1).unwrap().unwrap()
+						},
+						state_root: Default::default(),
+						digest: Default::default(),
+						extrinsics_root: Default::default(),
+					};
+
+					op.set_block_data(
+						header,
+						Some(vec![]),
+						None,
+						NewBlockState::Best,
+					).unwrap();
+					db.commit_operation(op).unwrap();
+				}
+
+				assert!(db.blockchain().hash(i).unwrap().is_some())
 			}
+			db.storage.db.clone()
+		};
 
-			assert!(db.blockchain().hash(i).unwrap().is_some())
+		let backend = Backend::<Block>::from_kvdb(backing, PruningMode::keep_blocks(1), 0).unwrap();
+		assert_eq!(backend.blockchain().info().unwrap().best_number, 9);
+		for i in 0..10 {
+			assert!(backend.blockchain().hash(i).unwrap().is_some())
 		}
 	}
 

core/client/db/src/light.rs

@@ -196,7 +196,8 @@ impl<Block: BlockT> LightStorage<Block> {
 			).into())
 		}
 
-		transaction.put(columns::META, meta_keys::FINALIZED_BLOCK, hash.as_ref());
+		let lookup_key = ::utils::number_to_lookup_key(header.number().clone());
+		transaction.put(columns::META, meta_keys::FINALIZED_BLOCK, &lookup_key);
 
 		// build new CHT if required
 		if let Some(new_cht_number) = cht::is_build_required(cht::SIZE, *header.number()) {
@@ -244,6 +245,14 @@ impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
 		let number = *header.number();
 		let parent_hash = *header.parent_hash();
 
+		// blocks in longest chain are keyed by number
+		let lookup_key = if leaf_state.is_best() {
+			::utils::number_to_lookup_key(number).to_vec()
+		} else {
+		// other blocks are keyed by number + hash
+			::utils::number_and_hash_to_lookup_key(number, hash)
+		};
+
 		if leaf_state.is_best() {
 			// handle reorg.
 			{
@@ -298,17 +307,9 @@ impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
 				}
 			}
 
-			transaction.put(columns::META, meta_keys::BEST_BLOCK, hash.as_ref());
+			transaction.put(columns::META, meta_keys::BEST_BLOCK, &lookup_key);
 		}
 
-		// blocks in longest chain are keyed by number
-		let lookup_key = if leaf_state.is_best() {
-			::utils::number_to_lookup_key(number).to_vec()
-		} else {
-		// other blocks are keyed by number + hash
-			::utils::number_and_hash_to_lookup_key(number, hash)
-		};
-
 		transaction.put(columns::HEADER, &lookup_key, &header.encode());
 		transaction.put(columns::HASH_LOOKUP, hash.as_ref(), &lookup_key);
 

core/client/db/src/utils.rs

@@ -53,6 +53,7 @@ pub mod meta_keys {
 }
 
 /// Database metadata.
+#[derive(Debug)]
 pub struct Meta<N, H> {
 	/// Hash of the best known block.
 	pub best_hash: H,

core/client/src/call_executor.rs

@@ -150,7 +150,7 @@ where
 		let heap_pages = state.storage(well_known_keys::HEAP_PAGES)
 			.map_err(|e| error::ErrorKind::Execution(Box::new(e)))?
 			.and_then(|v| u64::decode(&mut &v[..]))
-			.unwrap_or(8) as usize;
+			.unwrap_or(1024) as usize;
 
 		let mut ext = Ext::new(&mut overlay, &state, self.backend.changes_trie_storage());
 		self.executor.runtime_version(&mut ext, heap_pages, &code)