Merge #773: HF: Dyanfed pruned header now has extra root

fd5ec4b HF: Simplify legacy->dynafed transition max sig size (Gregory Sanders)
7b15b48 HF: Publish full dynafed block no matter epoch age (Gregory Sanders)
1d78473 HF: Dyanfed pruned header now has extra root (Gregory Sanders)

Pull request description:

  andytoshi's motivating text:

  "Motivation here is that the functionary wants to be able to compute the root of the parameters and use this to look up the params in the set of supported params (which includes a bunch of extra auxiliary data, in particular the signblock witness script).

  If the root is computed differently for compact-encoded params than for full-encoded ones, and indeed does not even commit to the extra data, this lookup is impossible without blockchain context, which is not obvious how to obtain for an unsigned block."

Tree-SHA512: 737b9da99bc4abd28cc63c3a441098e0a6ea0cb2748082f845b24d50fe3a9cd1d1253d622cdca4df91e27c04a4912babc8a54409e7e0f2af14f65a3414a072ad

src/chainparams.cpp

@@ -720,7 +720,8 @@ class CLiquidV1Params : public CChainParams {
         // Block signing encumberance script, default of 51 aka OP_TRUE
         std::vector<unsigned char> sign_bytes = ParseHex("5b21026a2a106ec32c8a1e8052e5d02a7b0a150423dbd9b116fc48d46630ff6e6a05b92102791646a8b49c2740352b4495c118d876347bf47d0551c01c4332fdc2df526f1a2102888bda53a424466b0451627df22090143bbf7c060e9eacb1e38426f6b07f2ae12102aee8967150dee220f613de3b239320355a498808084a93eaf39a34dcd62024852102d46e9259d0a0bb2bcbc461a3e68f34adca27b8d08fbe985853992b4b104e27412102e9944e35e5750ab621e098145b8e6cf373c273b7c04747d1aa020be0af40ccd62102f9a9d4b10a6d6c56d8c955c547330c589bb45e774551d46d415e51cd9ad5116321033b421566c124dfde4db9defe4084b7aa4e7f36744758d92806b8f72c2e943309210353dcc6b4cf6ad28aceb7f7b2db92a4bf07ac42d357adf756f3eca790664314b621037f55980af0455e4fb55aad9b85a55068bb6dc4740ea87276dc693f4598db45fa210384001daa88dabd23db878dbb1ce5b4c2a5fa72c3113e3514bf602325d0c37b8e21039056d089f2fe72dbc0a14780b4635b0dc8a1b40b7a59106325dd1bc45cc70493210397ab8ea7b0bf85bc7fc56bb27bf85e75502e94e76a6781c409f3f2ec3d1122192103b00e3b5b77884bf3cae204c4b4eac003601da75f96982ffcb3dcb29c5ee419b92103c1f3c0874cfe34b8131af34699589aacec4093399739ae352e8a46f80a6f68375fae");
         consensus.signblockscript = CScript(sign_bytes.begin(), sign_bytes.end());
-        consensus.max_block_signature_size = 12*74; // 11 signatures plus wiggle room
+        // 11 signatures, 15 pubkeys, plus wiggle room
+        consensus.max_block_signature_size = 12*74+16*33;
         g_signed_blocks = true;
 
         g_con_blockheightinheader = true;

src/dynafed.cpp

@@ -73,7 +73,7 @@ DynaFedParamEntry ComputeNextBlockFullCurrentParameters(const CBlockIndex* pinde
         CScript sh_wsh_fedpeg_program = CScript() << OP_HASH160 << ToByteVector(fedpeg_p2sh) << OP_EQUAL;
 
         // Put them in winning proposal
-        winning_proposal = DynaFedParamEntry(p2wsh_signblock_script, consensus.max_block_signature_size+consensus.signblockscript.size(), sh_wsh_fedpeg_program, consensus.fedpegScript, consensus.first_extension_space);
+        winning_proposal = DynaFedParamEntry(p2wsh_signblock_script, consensus.max_block_signature_size, sh_wsh_fedpeg_program, consensus.fedpegScript, consensus.first_extension_space);
     } else {
         winning_proposal = p_epoch_start->dynafed_params.m_current;
     }
@@ -91,12 +91,13 @@ DynaFedParamEntry ComputeNextBlockCurrentParameters(const CBlockIndex* pindexPre
     const uint32_t epoch_length = consensus.dynamic_epoch_length;
     uint32_t epoch_age = next_height % epoch_length;
 
-    // Return appropriate format based on epoch age
-    if (epoch_age > 0) {
-        // TODO implement "prune" function to remove fields in place and change serialize type
-        return DynaFedParamEntry(entry.m_signblockscript, entry.m_signblock_witness_limit);
-    } else {
+    // Return appropriate format based on epoch age or if we *just* activated
+    // dynafed via BIP9
+    if (epoch_age == 0 || pindexPrev->dynafed_params.IsNull()) {
         return entry;
+    } else {
+        return DynaFedParamEntry(entry.m_signblockscript, entry.m_signblock_witness_limit, entry.CalculateExtraRoot());
+
     }
 }
 

src/miner.cpp

@@ -151,8 +151,8 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc
     fIncludeWitness = IsWitnessEnabled(pindexPrev, chainparams.GetConsensus());
 
     if (IsDynaFedEnabled(pindexPrev, chainparams.GetConsensus())) {
-        DynaFedParamEntry current_params = ComputeNextBlockCurrentParameters(chainActive.Tip(), chainparams.GetConsensus());
-        DynaFedParams block_params(current_params, proposed_entry ? *proposed_entry : DynaFedParamEntry());
+        const DynaFedParamEntry current_params = ComputeNextBlockCurrentParameters(chainActive.Tip(), chainparams.GetConsensus());
+        const DynaFedParams block_params(current_params, proposed_entry ? *proposed_entry : DynaFedParamEntry());
         pblock->m_dynafed_params = block_params;
         nBlockWeight += ::GetSerializeSize(block_params, PROTOCOL_VERSION)*WITNESS_SCALE_FACTOR;
         nBlockWeight += current_params.m_signblock_witness_limit; // Note witness discount

src/primitives/block.cpp

@@ -43,19 +43,39 @@ std::string CBlock::ToString() const
 
 uint256 DynaFedParamEntry::CalculateRoot() const
 {
-    if (IsNull()) {
+    if (m_serialize_type == 0) {
         return uint256();
     }
 
+    std::vector<uint256> compact_leaves;
+    compact_leaves.push_back(SerializeHash(m_signblockscript, SER_GETHASH, 0));
+    compact_leaves.push_back(SerializeHash(m_signblock_witness_limit, SER_GETHASH, 0));
+    uint256 compact_root(ComputeFastMerkleRoot(compact_leaves));
+
+    uint256 extra_root;
+    if (m_serialize_type ==1 ) {
+        // It's pruned, take the stored value
+        extra_root = m_elided_root;
+    } else if (m_serialize_type == 2) {
+        // It's unpruned, compute the node value
+        extra_root = CalculateExtraRoot();
+    }
+
     std::vector<uint256> leaves;
-    leaves.push_back(SerializeHash(m_signblockscript, SER_GETHASH, 0));
-    leaves.push_back(SerializeHash(m_signblock_witness_limit, SER_GETHASH, 0));
-    leaves.push_back(SerializeHash(m_fedpeg_program, SER_GETHASH, 0));
-    leaves.push_back(SerializeHash(m_fedpegscript, SER_GETHASH, 0));
-    leaves.push_back(SerializeHash(m_extension_space, SER_GETHASH, 0));
+    leaves.push_back(compact_root);
+    leaves.push_back(extra_root);
     return ComputeFastMerkleRoot(leaves);
 }
 
+uint256 DynaFedParamEntry::CalculateExtraRoot() const
+{
+    std::vector<uint256> extra_leaves;
+    extra_leaves.push_back(SerializeHash(m_fedpeg_program, SER_GETHASH, 0));
+    extra_leaves.push_back(SerializeHash(m_fedpegscript, SER_GETHASH, 0));
+    extra_leaves.push_back(SerializeHash(m_extension_space, SER_GETHASH, 0));
+    return ComputeFastMerkleRoot(extra_leaves);
+}
+
 uint256 DynaFedParams::CalculateRoot() const
 {
     if (IsNull()) {

src/primitives/block.h

@@ -58,18 +58,25 @@ class CProof
 class DynaFedParamEntry
 {
 public:
-    unsigned char m_serialize_type; // Determines how it is serialized, defaults to null
+    // Determines how these entries are serialized and stored
+    // 0 -> Null. Only used for proposed parameter "null votes"
+    // 1 -> Pruned. Doesn't have non-signblockscript data. That elided data
+    // is committed to in m_elided_root, and validated against chainstate.
+    // 2 -> Full. Typically only consensus-legal at epoch start.
+    unsigned char m_serialize_type;
+
     CScript m_signblockscript;
     uint32_t m_signblock_witness_limit; // Max block signature witness serialized size
     CScript m_fedpeg_program; // The "scriptPubKey" of the fedpegscript
     CScript m_fedpegscript; // The witnessScript for witness v0 or undefined otherwise.
     // No consensus meaning to the particular bytes, currently we interpret as PAK keys, details in pak.h
     std::vector<std::vector<unsigned char>> m_extension_space;
+    uint256 m_elided_root; // non-zero only when m_serialize_type == 1
 
     // Each constructor sets its own serialization type implicitly based on which
     // arguments are given
     DynaFedParamEntry() { m_signblock_witness_limit = 0; m_serialize_type = 0; };
-    DynaFedParamEntry(const CScript& signblockscript_in, const uint32_t sbs_wit_limit_in) : m_signblockscript(signblockscript_in), m_signblock_witness_limit(sbs_wit_limit_in) { m_serialize_type = 1; };
+    DynaFedParamEntry(const CScript& signblockscript_in, const uint32_t sbs_wit_limit_in, const uint256 elided_root_in) : m_signblockscript(signblockscript_in), m_signblock_witness_limit(sbs_wit_limit_in), m_elided_root(elided_root_in) { m_serialize_type = 1; };
     DynaFedParamEntry(const CScript& signblockscript_in, const uint32_t sbs_wit_limit_in, const CScript& fedpeg_program_in, const CScript& fedpegscript_in, const std::vector<std::vector<unsigned char>> extension_space_in) : m_signblockscript(signblockscript_in), m_signblock_witness_limit(sbs_wit_limit_in), m_fedpeg_program(fedpeg_program_in), m_fedpegscript(fedpegscript_in), m_extension_space(extension_space_in) { m_serialize_type = 2; };
 
     ADD_SERIALIZE_METHODS;
@@ -84,6 +91,7 @@ class DynaFedParamEntry
             case 1:
                 READWRITE(m_signblockscript);
                 READWRITE(m_signblock_witness_limit);
+                READWRITE(m_elided_root);
                 break;
             case 2:
                 READWRITE(m_signblockscript);
@@ -98,6 +106,8 @@ class DynaFedParamEntry
     }
 
     uint256 CalculateRoot() const;
+    // Calculates root for the non-blocksigning merkle fields
+    uint256 CalculateExtraRoot() const;
 
     bool IsNull() const
     {

src/test/dynafed_tests.cpp

@@ -21,23 +21,23 @@ BOOST_AUTO_TEST_CASE(dynafed_params_root)
     CScript fp_script(opcodetype(4));
     std::vector<std::vector<unsigned char>> ext{ {5, 6}, {7} };
 
-    DynaFedParamEntry compact_entry = DynaFedParamEntry(signblockscript, signblock_wl);
+    DynaFedParamEntry compact_entry = DynaFedParamEntry(signblockscript, signblock_wl, uint256());
     BOOST_CHECK_EQUAL(
         compact_entry.CalculateRoot().GetHex(),
-        "dff5f3793abc06a6d75e80fe3cfd47406f732fa4ec9305960ae2a229222a1ad5"
+        "f98f149fd11da6fbe26d0ee53cadd28372fa9eed2cb7080f41da7ca311531777"
     );
 
     DynaFedParamEntry full_entry =
         DynaFedParamEntry(signblockscript, signblock_wl, fp_program, fp_script, ext);
     BOOST_CHECK_EQUAL(
         full_entry.CalculateRoot().GetHex(),
-        "175be2087ba7cc0e33348bef493bd3e34f31f64bf9226e5881ab310dafa432ff"
+        "8eb1b83cce69a3d8b0bfb7fbe77ae8f1d24b57a9cae047b8c0aba084ad878249"
     );
 
     DynaFedParams params = DynaFedParams(compact_entry, full_entry);
     BOOST_CHECK_EQUAL(
         params.CalculateRoot().GetHex(),
-        "e56cf79487952dfa85fe6a85829600adc19714ba6ab1157fdff02b25ae60cee2"
+        "113160f76dc17fe367a2def79aefe06feeea9c795310c9e88aeedc23e145982e"
     );
 }
 

test/functional/feature_blocksign.py

@@ -84,7 +84,7 @@ def set_test_params(self):
         self.witnessScript = signblockscript # post-dynafed this becomes witnessScript
         self.extra_args = [[
             "-signblockscript={}".format(signblockscript),
-            "-con_max_block_sig_size={}".format(self.required_signers*74),
+            "-con_max_block_sig_size={}".format(self.required_signers*74+self.num_nodes*33),
             "-anyonecanspendaremine=1",
             "-con_dyna_deploy_start=0",
         ]] * self.num_nodes
@@ -218,14 +218,16 @@ def run_test(self):
 
         # Next let's activate dynafed
         blocks_til_dynafed = 431 - self.nodes[0].getblockcount()
+        self.log.info("Activating dynafed")
         self.mine_blocks(blocks_til_dynafed, False)
         self.check_height(111+blocks_til_dynafed)
 
         assert_equal(self.nodes[0].getblockchaininfo()['bip9_softforks']['dynafed']['status'], "active")
 
-        self.log.info("Mine some dynamic federation blocks without and with txns")
-        self.mine_blocks(50, False)
-        self.mine_blocks(50, True)
+        self.log.info("Mine some dynamic federation blocks without txns")
+        self.mine_blocks(10, False)
+        self.log.info("Mine some dynamic federation blocks with txns")
+        self.mine_blocks(10, True)
 
 if __name__ == '__main__':
     BlockSignTest().main()

test/functional/feature_dynafed.py

@@ -34,7 +34,7 @@ def go_to_epoch_end(node):
     blocks_to_mine = epoch_info["epoch_length"] - epoch_info["epoch_age"] - 1
     node.generatetoaddress(blocks_to_mine, node.getnewaddress())
 
-def validate_no_vote_op_true(node, block):
+def validate_no_vote_op_true(node, block, first_dynafed_active_block):
 
     block_info = node.getblock(block)
     dynamic_parameters = block_info["dynamic_parameters"]
@@ -44,14 +44,13 @@ def validate_no_vote_op_true(node, block):
     # signblockscript is now the P2WSH-ification of OP_TRUE
     WSH_OP_TRUE = node.decodescript("51")["segwit"]["hex"]
     assert_equal(dynamic_parameters["current"]["signblockscript"], WSH_OP_TRUE)
-    if block_height % 10 == 0:
+    if block_height % 10 == 0 or first_dynafed_active_block:
         assert_equal(dynamic_parameters["current"]["fedpegscript"], "51")
         assert_equal(dynamic_parameters["current"]["extension_space"], initial_extension)
     else:
         assert_equal(dynamic_parameters["current"]["fedpegscript"], "")
         assert_equal(dynamic_parameters["current"]["extension_space"], [])
-    # TODO workshop this bump, or commit to new value in chainparams instead
-    assert_equal(dynamic_parameters["current"]["max_block_witness"], 75)
+    assert_equal(dynamic_parameters["current"]["max_block_witness"], 74)
     # nothing was proposed, null fields make impossible to be valid blockheader
     # due to script rules requiring bool true on stack
     assert_equal(dynamic_parameters["proposed"]["signblockscript"], "")
@@ -118,8 +117,9 @@ def test_dynafed_activation(self):
         # Next block is first dynamic federation block
         block = self.nodes[0].generatetoaddress(1, self.nodes[0].getnewaddress())[0]
         self.sync_all()
+        # We publish full block on BIP9 transition
         for i in range(self.num_nodes):
-            validate_no_vote_op_true(self.nodes[i], block)
+            validate_no_vote_op_true(self.nodes[i], block, True)
 
     def test_illegal_proposals(self):
 
@@ -148,14 +148,14 @@ def test_no_vote(self):
 
         for i in range(self.num_nodes):
             for block in blocks:
-                validate_no_vote_op_true(self.nodes[i], block)
+                validate_no_vote_op_true(self.nodes[i], block, False)
 
         # Now transition using vanilla getnewblockhex, nothing changed
         block = self.nodes[0].generatetoaddress(1, self.nodes[0].getnewaddress())[0]
         self.sync_all()
 
         for i in range(self.num_nodes):
-            validate_no_vote_op_true(self.nodes[i], block)
+            validate_no_vote_op_true(self.nodes[i], block, False)
 
     def test_under_vote(self):
         self.log.info("Testing failed voting epoch...")
@@ -176,7 +176,7 @@ def test_under_vote(self):
         self.sync_all()
 
         for i in range(self.num_nodes):
-            validate_no_vote_op_true(self.nodes[i], block)
+            validate_no_vote_op_true(self.nodes[i], block, False)
 
     def test_four_fifth_vote(self):
         self.log.info("Testing just-successful transition epoch...")
@@ -192,7 +192,7 @@ def test_four_fifth_vote(self):
                 chain_info = self.nodes[i].getblockchaininfo()
                 fedpeg_info = self.nodes[i].getsidechaininfo()
                 assert_equal(chain_info["current_signblock_hex"], WSH_OP_TRUE)
-                assert_equal(chain_info["max_block_witness"], 75)
+                assert_equal(chain_info["max_block_witness"], 74)
                 assert_equal(chain_info["extension_space"], initial_extension)
                 assert_equal(fedpeg_info["current_fedpegscripts"], ["51", "51"])
 
@@ -210,7 +210,7 @@ def test_four_fifth_vote(self):
             chain_info = self.nodes[i].getblockchaininfo()
             fedpeg_info = self.nodes[i].getsidechaininfo()
             assert_equal(chain_info["current_signblock_hex"], WSH_OP_TRUE)
-            assert_equal(chain_info["max_block_witness"], 75)
+            assert_equal(chain_info["max_block_witness"], 74)
             assert_equal(chain_info["extension_space"], initial_extension)
             assert_equal(fedpeg_info["current_fedpegscripts"], ["51", "51"])
 

test/functional/test_framework/messages.py

@@ -836,22 +836,24 @@ def __repr__(self):
             % (self.challenge, self.solution)
 
 class DynaFedParamEntry:
-    __slots__ = ("m_serialize_type", "m_signblockscript", "m_signblock_witness_limit", "m_fedpeg_program", "m_fedpegscript", "m_extension_space")
+    __slots__ = ("m_serialize_type", "m_signblockscript", "m_signblock_witness_limit", "m_fedpeg_program", "m_fedpegscript", "m_extension_space", "m_elided_root")
 
     # Constructor args will define serialization type:
     # null = 0
     # signblock-related fields = 1, required for m_current on non-epoch-starts
     # all fields = 2, required for epoch starts
-    def __init__(self, m_signblockscript=b"", m_signblock_witness_limit=0, m_fedpeg_program=b"", m_fedpegscript=b"", m_extension_space=[]):
+    def __init__(self, m_signblockscript=b"", m_signblock_witness_limit=0, m_fedpeg_program=b"", m_fedpegscript=b"", m_extension_space=[], m_elided_root=0):
         self.m_signblockscript = m_signblockscript
         self.m_signblock_witness_limit = m_signblock_witness_limit
         self.m_fedpeg_program = m_fedpeg_program
         self.m_fedpegscript = m_fedpegscript
         self.m_extension_space = m_extension_space
         if self.is_null():
             self.m_serialize_type = 0
-        elif m_fedpegscript==b"" and m_extension_space == []:
+        elif m_fedpegscript==b"" and m_fedpeg_program==b"" and m_extension_space == []:
             self.m_serialize_type = 1
+            # We also set the "extra root" in this case
+            self.m_elided_root = m_elided_root
         else:
             self.m_serialize_type = 2
 
@@ -862,6 +864,7 @@ def set_null(self):
         self.m_fedpegscript = b""
         self.m_extension_space = []
         self.m_serialize_type = 0
+        self.m_elided_root = 0
 
     def is_null(self):
         return self.m_signblockscript == b"" and self.m_signblock_witness_limit == 0 and \
@@ -874,6 +877,7 @@ def serialize(self):
         if self.m_serialize_type == 1:
             r += ser_string(self.m_signblockscript)
             r += struct.pack("<I", self.m_signblock_witness_limit)
+            r += ser_uint256(self.m_elided_root)
         elif self.m_serialize_type == 2:
             r += ser_string(self.m_signblockscript)
             r += struct.pack("<I", self.m_signblock_witness_limit)
@@ -889,6 +893,7 @@ def deserialize(self, f):
         if self.m_serialize_type == 1:
             self.m_signblockscript = deser_string(f)
             self.m_signblock_witness_limit = struct.unpack("<I", f.read(4))[0]
+            self.m_elided_root = deser_uint256(f)
         elif self.m_serialize_type == 2:
             self.m_signblockscript = deser_string(f)
             self.m_signblock_witness_limit = struct.unpack("<I", f.read(4))[0]